2024-03-07T02:37:55.123 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-07T02:37:55.123 DEBUG:teuthology.run:Teuthology command: teuthology --verbose --name teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780 --owner scheduled_teuthology@teuthology --description upgrade:reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780/orig.config.yaml 2024-03-07T02:37:55.181 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-07T02:37:55.277 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780 branch: main description: upgrade:reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7583780' kernel: kdb: true sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 workunit: branch: main sha1: ac30ee6cd9cb289983b4db31a5b462d33a1efd30 owner: scheduled_teuthology@teuthology priority: 50 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 1775 sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:reef-x suite_branch: main suite_path: /home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: ac30ee6cd9cb289983b4db31a5b462d33a1efd30 targets: smithi148.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBA4zgKKJtKEHB2ABRlRQ0xVhJ18MtIh0fh6AfF4DkHvr0jeQV9eHB44QqGWqgac5b4r93UhBb3Es3CRRsyen9IQ= smithi195.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM69fqkVao/QzEKb9dWEj4UEZOWwl5KaihGNOJXzaIZxOxle2qemPgCKNrf1fsSPG2TOFVC33onxYsRglMh532g= tasks: - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-07_02:25:02 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-07T02:37:55.277 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa; will attempt to use it 2024-03-07T02:37:55.278 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks 2024-03-07T02:37:55.278 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-07T02:37:55.280 INFO:teuthology.task.internal:Checking packages... 2024-03-07T02:37:55.303 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71' 2024-03-07T02:37:55.303 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:37:55.303 INFO:teuthology.packaging:ref: None 2024-03-07T02:37:55.303 INFO:teuthology.packaging:tag: None 2024-03-07T02:37:55.303 INFO:teuthology.packaging:branch: main 2024-03-07T02:37:55.303 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:37:55.304 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=main 2024-03-07T02:37:55.731 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1905-gea9a5688-1jammy 2024-03-07T02:37:55.733 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-07T02:37:55.739 INFO:teuthology.task.internal:no buildpackages task found 2024-03-07T02:37:55.739 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-07T02:37:55.744 INFO:teuthology.task.internal:Saving configuration 2024-03-07T02:37:55.766 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-07T02:37:55.771 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-07T02:37:55.796 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi148.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:27.937163', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAtBHbh7ErkFfFuj8a9busxddhya3dU8s83LEkJkAUtF'} 2024-03-07T02:37:55.819 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi195.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:27.938540', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDDCYy0a4VMYvu8Y5a4TbbA5iAy/J3XeEtxd3Jw0ZEen'} 2024-03-07T02:37:55.819 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-07T02:37:55.825 INFO:teuthology.task.internal:roles: ubuntu@smithi148.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-03-07T02:37:55.825 INFO:teuthology.task.internal:roles: ubuntu@smithi195.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-03-07T02:37:55.825 INFO:teuthology.run_tasks:Running task console_log... 2024-03-07T02:37:55.895 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f23c42dd1f0>, signals=[15]) 2024-03-07T02:37:55.895 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-07T02:37:55.901 INFO:teuthology.task.internal:Opening connections... 2024-03-07T02:37:55.901 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi148.front.sepia.ceph.com 2024-03-07T02:37:55.903 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:37:55.981 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi195.front.sepia.ceph.com 2024-03-07T02:37:55.982 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:37:56.060 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-07T02:37:56.066 DEBUG:teuthology.orchestra.run.smithi148:> uname -m 2024-03-07T02:37:56.072 INFO:teuthology.orchestra.run.smithi148.stdout:x86_64 2024-03-07T02:37:56.072 DEBUG:teuthology.orchestra.run.smithi148:> cat /etc/os-release 2024-03-07T02:37:56.119 INFO:teuthology.orchestra.run.smithi148.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:NAME="Ubuntu" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:VERSION_ID="22.04" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:VERSION_CODENAME=jammy 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:ID=ubuntu 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:ID_LIKE=debian 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:HOME_URL="https://www.ubuntu.com/" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-03-07T02:37:56.120 INFO:teuthology.orchestra.run.smithi148.stdout:UBUNTU_CODENAME=jammy 2024-03-07T02:37:56.121 INFO:teuthology.lock.ops:Updating smithi148.front.sepia.ceph.com on lock server 2024-03-07T02:37:56.141 DEBUG:teuthology.orchestra.run.smithi195:> uname -m 2024-03-07T02:37:56.147 INFO:teuthology.orchestra.run.smithi195.stdout:x86_64 2024-03-07T02:37:56.147 DEBUG:teuthology.orchestra.run.smithi195:> cat /etc/os-release 2024-03-07T02:37:56.192 INFO:teuthology.orchestra.run.smithi195.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-03-07T02:37:56.192 INFO:teuthology.orchestra.run.smithi195.stdout:NAME="Ubuntu" 2024-03-07T02:37:56.192 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION_ID="22.04" 2024-03-07T02:37:56.192 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-03-07T02:37:56.192 INFO:teuthology.orchestra.run.smithi195.stdout:VERSION_CODENAME=jammy 2024-03-07T02:37:56.193 INFO:teuthology.orchestra.run.smithi195.stdout:ID=ubuntu 2024-03-07T02:37:56.193 INFO:teuthology.orchestra.run.smithi195.stdout:ID_LIKE=debian 2024-03-07T02:37:56.193 INFO:teuthology.orchestra.run.smithi195.stdout:HOME_URL="https://www.ubuntu.com/" 2024-03-07T02:37:56.193 INFO:teuthology.orchestra.run.smithi195.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-03-07T02:37:56.193 INFO:teuthology.orchestra.run.smithi195.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-03-07T02:37:56.193 INFO:teuthology.orchestra.run.smithi195.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-03-07T02:37:56.193 INFO:teuthology.orchestra.run.smithi195.stdout:UBUNTU_CODENAME=jammy 2024-03-07T02:37:56.193 INFO:teuthology.lock.ops:Updating smithi195.front.sepia.ceph.com on lock server 2024-03-07T02:37:56.217 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-07T02:37:56.225 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-07T02:37:56.230 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-07T02:37:56.231 DEBUG:teuthology.orchestra.run.smithi148:> test '!' -e /home/ubuntu/cephtest 2024-03-07T02:37:56.233 DEBUG:teuthology.orchestra.run.smithi195:> test '!' -e /home/ubuntu/cephtest 2024-03-07T02:37:56.240 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-07T02:37:56.246 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-07T02:37:56.246 DEBUG:teuthology.orchestra.run.smithi148:> test -z $(ls -A /var/lib/ceph) 2024-03-07T02:37:56.281 DEBUG:teuthology.orchestra.run.smithi195:> test -z $(ls -A /var/lib/ceph) 2024-03-07T02:37:56.380 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-07T02:37:56.439 INFO:teuthology.run_tasks:Running task kernel... 2024-03-07T02:37:56.456 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-07T02:37:56.456 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-07T02:37:56.457 DEBUG:teuthology.orchestra.run.smithi148:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-07T02:37:56.457 DEBUG:teuthology.orchestra.run.smithi195:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-07T02:37:56.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T02:37:56.462 DEBUG:teuthology.orchestra.run.smithi148:> uname -r 2024-03-07T02:37:56.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T02:37:56.463 DEBUG:teuthology.orchestra.run.smithi195:> uname -r 2024-03-07T02:37:56.508 INFO:teuthology.orchestra.run.smithi148.stdout:5.15.0-56-generic 2024-03-07T02:37:56.508 INFO:teuthology.task.kernel:Running kernel on smithi148: 5.15.0-56-generic 2024-03-07T02:37:56.508 DEBUG:teuthology.orchestra.run.smithi148:> sudo apt-get clean 2024-03-07T02:37:56.509 INFO:teuthology.orchestra.run.smithi195.stdout:5.15.0-56-generic 2024-03-07T02:37:56.509 INFO:teuthology.task.kernel:Running kernel on smithi195: 5.15.0-56-generic 2024-03-07T02:37:56.510 DEBUG:teuthology.orchestra.run.smithi195:> sudo apt-get clean 2024-03-07T02:37:56.640 DEBUG:teuthology.orchestra.run.smithi195:> sudo apt-get update 2024-03-07T02:37:56.653 DEBUG:teuthology.orchestra.run.smithi148:> sudo apt-get update 2024-03-07T02:37:56.903 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:37:56.911 INFO:teuthology.orchestra.run.smithi195.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB] 2024-03-07T02:37:56.911 INFO:teuthology.orchestra.run.smithi195.stdout:Get:3 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB] 2024-03-07T02:37:56.913 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:37:56.913 INFO:teuthology.orchestra.run.smithi148.stdout:Get:2 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB] 2024-03-07T02:37:56.976 INFO:teuthology.orchestra.run.smithi148.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB] 2024-03-07T02:37:57.053 INFO:teuthology.orchestra.run.smithi195.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB] 2024-03-07T02:37:57.162 INFO:teuthology.orchestra.run.smithi195.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [415 kB] 2024-03-07T02:37:57.252 INFO:teuthology.orchestra.run.smithi195.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,433 kB] 2024-03-07T02:37:57.333 INFO:teuthology.orchestra.run.smithi148.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,217 kB] 2024-03-07T02:37:57.337 INFO:teuthology.orchestra.run.smithi148.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB] 2024-03-07T02:37:57.373 INFO:teuthology.orchestra.run.smithi195.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,217 kB] 2024-03-07T02:37:57.471 INFO:teuthology.orchestra.run.smithi195.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [581 kB] 2024-03-07T02:37:57.476 INFO:teuthology.orchestra.run.smithi195.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [220 kB] 2024-03-07T02:37:57.477 INFO:teuthology.orchestra.run.smithi148.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [581 kB] 2024-03-07T02:37:57.485 INFO:teuthology.orchestra.run.smithi195.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB] 2024-03-07T02:37:57.485 INFO:teuthology.orchestra.run.smithi195.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,495 kB] 2024-03-07T02:37:57.488 INFO:teuthology.orchestra.run.smithi195.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [280 kB] 2024-03-07T02:37:57.500 INFO:teuthology.orchestra.run.smithi195.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB] 2024-03-07T02:37:57.501 INFO:teuthology.orchestra.run.smithi195.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,523 kB] 2024-03-07T02:37:57.520 INFO:teuthology.orchestra.run.smithi195.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [33.7 kB] 2024-03-07T02:37:57.521 INFO:teuthology.orchestra.run.smithi195.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [248 kB] 2024-03-07T02:37:57.543 INFO:teuthology.orchestra.run.smithi195.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:37:57.543 INFO:teuthology.orchestra.run.smithi195.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [35.3 kB] 2024-03-07T02:37:57.543 INFO:teuthology.orchestra.run.smithi195.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [596 kB] 2024-03-07T02:37:57.544 INFO:teuthology.orchestra.run.smithi195.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [253 kB] 2024-03-07T02:37:57.549 INFO:teuthology.orchestra.run.smithi195.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:37:57.549 INFO:teuthology.orchestra.run.smithi195.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,053 kB] 2024-03-07T02:37:57.553 INFO:teuthology.orchestra.run.smithi195.stdout:Get:23 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [847 kB] 2024-03-07T02:37:57.568 INFO:teuthology.orchestra.run.smithi195.stdout:Get:24 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [161 kB] 2024-03-07T02:37:57.571 INFO:teuthology.orchestra.run.smithi195.stdout:Get:25 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB] 2024-03-07T02:37:57.571 INFO:teuthology.orchestra.run.smithi195.stdout:Get:26 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-03-07T02:37:57.571 INFO:teuthology.orchestra.run.smithi195.stdout:Get:27 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.1 kB] 2024-03-07T02:37:57.582 INFO:teuthology.orchestra.run.smithi195.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [692 kB] 2024-03-07T02:37:57.603 INFO:teuthology.orchestra.run.smithi195.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [237 kB] 2024-03-07T02:37:57.604 INFO:teuthology.orchestra.run.smithi195.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,476 B] 2024-03-07T02:37:57.605 INFO:teuthology.orchestra.run.smithi195.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B] 2024-03-07T02:37:57.608 INFO:teuthology.orchestra.run.smithi195.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB] 2024-03-07T02:37:57.608 INFO:teuthology.orchestra.run.smithi195.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [42.1 kB] 2024-03-07T02:37:57.609 INFO:teuthology.orchestra.run.smithi195.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,160 B] 2024-03-07T02:37:57.615 INFO:teuthology.orchestra.run.smithi195.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.1 kB] 2024-03-07T02:37:57.615 INFO:teuthology.orchestra.run.smithi195.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B] 2024-03-07T02:37:57.616 INFO:teuthology.orchestra.run.smithi195.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [41.7 kB] 2024-03-07T02:37:57.617 INFO:teuthology.orchestra.run.smithi195.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [33.9 kB] 2024-03-07T02:37:57.629 INFO:teuthology.orchestra.run.smithi195.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [10.5 kB] 2024-03-07T02:37:57.630 INFO:teuthology.orchestra.run.smithi195.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-03-07T02:37:57.630 INFO:teuthology.orchestra.run.smithi195.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [13.4 kB] 2024-03-07T02:37:57.631 INFO:teuthology.orchestra.run.smithi195.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [24.3 kB] 2024-03-07T02:37:57.631 INFO:teuthology.orchestra.run.smithi195.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-03-07T02:37:57.634 INFO:teuthology.orchestra.run.smithi195.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B] 2024-03-07T02:37:57.672 INFO:teuthology.orchestra.run.smithi148.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,433 kB] 2024-03-07T02:37:57.738 INFO:teuthology.orchestra.run.smithi148.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [415 kB] 2024-03-07T02:37:57.757 INFO:teuthology.orchestra.run.smithi148.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [220 kB] 2024-03-07T02:37:57.767 INFO:teuthology.orchestra.run.smithi148.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB] 2024-03-07T02:37:57.767 INFO:teuthology.orchestra.run.smithi148.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,495 kB] 2024-03-07T02:37:57.816 INFO:teuthology.orchestra.run.smithi148.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [280 kB] 2024-03-07T02:37:57.831 INFO:teuthology.orchestra.run.smithi148.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB] 2024-03-07T02:37:57.832 INFO:teuthology.orchestra.run.smithi148.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,523 kB] 2024-03-07T02:37:57.858 INFO:teuthology.orchestra.run.smithi148.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [33.7 kB] 2024-03-07T02:37:57.859 INFO:teuthology.orchestra.run.smithi148.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [248 kB] 2024-03-07T02:37:57.880 INFO:teuthology.orchestra.run.smithi148.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:37:57.881 INFO:teuthology.orchestra.run.smithi148.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [596 kB] 2024-03-07T02:37:57.891 INFO:teuthology.orchestra.run.smithi148.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [35.3 kB] 2024-03-07T02:37:57.892 INFO:teuthology.orchestra.run.smithi148.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [253 kB] 2024-03-07T02:37:57.893 INFO:teuthology.orchestra.run.smithi148.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [847 kB] 2024-03-07T02:37:57.898 INFO:teuthology.orchestra.run.smithi148.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B] 2024-03-07T02:37:58.063 INFO:teuthology.orchestra.run.smithi148.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [692 kB] 2024-03-07T02:37:58.079 INFO:teuthology.orchestra.run.smithi148.stdout:Get:24 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [161 kB] 2024-03-07T02:37:58.476 INFO:teuthology.orchestra.run.smithi148.stdout:Get:25 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB] 2024-03-07T02:37:58.483 INFO:teuthology.orchestra.run.smithi148.stdout:Get:26 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.1 kB] 2024-03-07T02:37:58.497 INFO:teuthology.orchestra.run.smithi148.stdout:Get:27 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-03-07T02:37:58.498 INFO:teuthology.orchestra.run.smithi148.stdout:Get:28 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,476 B] 2024-03-07T02:37:58.500 INFO:teuthology.orchestra.run.smithi148.stdout:Get:29 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B] 2024-03-07T02:37:58.640 INFO:teuthology.orchestra.run.smithi148.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,053 kB] 2024-03-07T02:37:58.750 INFO:teuthology.orchestra.run.smithi148.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [237 kB] 2024-03-07T02:37:58.771 INFO:teuthology.orchestra.run.smithi148.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB] 2024-03-07T02:37:58.772 INFO:teuthology.orchestra.run.smithi148.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [4,160 B] 2024-03-07T02:37:58.772 INFO:teuthology.orchestra.run.smithi148.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [42.1 kB] 2024-03-07T02:37:58.775 INFO:teuthology.orchestra.run.smithi148.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [10.1 kB] 2024-03-07T02:37:58.776 INFO:teuthology.orchestra.run.smithi148.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B] 2024-03-07T02:37:58.776 INFO:teuthology.orchestra.run.smithi148.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [41.7 kB] 2024-03-07T02:37:58.780 INFO:teuthology.orchestra.run.smithi148.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [33.9 kB] 2024-03-07T02:37:58.781 INFO:teuthology.orchestra.run.smithi148.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [10.5 kB] 2024-03-07T02:37:58.947 INFO:teuthology.orchestra.run.smithi148.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-03-07T02:37:59.029 INFO:teuthology.orchestra.run.smithi148.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [24.3 kB] 2024-03-07T02:37:59.135 INFO:teuthology.orchestra.run.smithi148.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [13.4 kB] 2024-03-07T02:37:59.163 INFO:teuthology.orchestra.run.smithi148.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.5 kB] 2024-03-07T02:37:59.212 INFO:teuthology.orchestra.run.smithi148.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B] 2024-03-07T02:38:03.228 INFO:teuthology.orchestra.run.smithi195.stdout:Fetched 12.0 MB in 2s (6,099 kB/s) 2024-03-07T02:38:03.472 INFO:teuthology.orchestra.run.smithi148.stdout:Fetched 12.0 MB in 3s (4,787 kB/s) 2024-03-07T02:38:04.505 INFO:teuthology.orchestra.run.smithi195.stdout:Reading package lists... 2024-03-07T02:38:04.522 DEBUG:teuthology.orchestra.run.smithi195:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:38:04.573 INFO:teuthology.orchestra.run.smithi195.stdout:Reading package lists... 2024-03-07T02:38:04.749 INFO:teuthology.orchestra.run.smithi195.stdout:Building dependency tree... 2024-03-07T02:38:04.749 INFO:teuthology.orchestra.run.smithi195.stdout:Reading state information... 2024-03-07T02:38:04.839 INFO:teuthology.orchestra.run.smithi148.stdout:Reading package lists... 2024-03-07T02:38:04.857 DEBUG:teuthology.orchestra.run.smithi148:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:38:04.898 INFO:teuthology.orchestra.run.smithi195.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:38:04.899 INFO:teuthology.orchestra.run.smithi195.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:38:04.899 INFO:teuthology.orchestra.run.smithi195.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:38:04.899 INFO:teuthology.orchestra.run.smithi195.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:38:04.899 INFO:teuthology.orchestra.run.smithi195.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-03-07T02:38:04.899 INFO:teuthology.orchestra.run.smithi195.stdout: python2.7-minimal 2024-03-07T02:38:04.899 INFO:teuthology.orchestra.run.smithi195.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:38:04.900 INFO:teuthology.orchestra.run.smithi195.stdout:The following additional packages will be installed: 2024-03-07T02:38:04.900 INFO:teuthology.orchestra.run.smithi195.stdout: linux-generic linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:38:04.900 INFO:teuthology.orchestra.run.smithi195.stdout: linux-headers-generic linux-image-5.15.0-97-generic 2024-03-07T02:38:04.901 INFO:teuthology.orchestra.run.smithi195.stdout: linux-modules-5.15.0-97-generic linux-modules-extra-5.15.0-97-generic 2024-03-07T02:38:04.902 INFO:teuthology.orchestra.run.smithi195.stdout:Suggested packages: 2024-03-07T02:38:04.902 INFO:teuthology.orchestra.run.smithi195.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-03-07T02:38:04.902 INFO:teuthology.orchestra.run.smithi195.stdout:Recommended packages: 2024-03-07T02:38:04.902 INFO:teuthology.orchestra.run.smithi195.stdout: thermald 2024-03-07T02:38:04.912 INFO:teuthology.orchestra.run.smithi148.stdout:Reading package lists... 2024-03-07T02:38:04.950 INFO:teuthology.orchestra.run.smithi195.stdout:The following NEW packages will be installed: 2024-03-07T02:38:04.950 INFO:teuthology.orchestra.run.smithi195.stdout: linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:38:04.951 INFO:teuthology.orchestra.run.smithi195.stdout: linux-image-5.15.0-97-generic linux-modules-5.15.0-97-generic 2024-03-07T02:38:04.951 INFO:teuthology.orchestra.run.smithi195.stdout: linux-modules-extra-5.15.0-97-generic 2024-03-07T02:38:04.952 INFO:teuthology.orchestra.run.smithi195.stdout:The following packages will be upgraded: 2024-03-07T02:38:04.952 INFO:teuthology.orchestra.run.smithi195.stdout: linux-generic linux-headers-generic linux-image-generic 2024-03-07T02:38:05.083 INFO:teuthology.orchestra.run.smithi195.stdout:3 upgraded, 5 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:38:05.083 INFO:teuthology.orchestra.run.smithi195.stdout:Need to get 113 MB of archives. 2024-03-07T02:38:05.083 INFO:teuthology.orchestra.run.smithi195.stdout:After this operation, 582 MB of additional disk space will be used. 2024-03-07T02:38:05.083 INFO:teuthology.orchestra.run.smithi195.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-97-generic amd64 5.15.0-97.107 [22.6 MB] 2024-03-07T02:38:05.104 INFO:teuthology.orchestra.run.smithi148.stdout:Building dependency tree... 2024-03-07T02:38:05.105 INFO:teuthology.orchestra.run.smithi148.stdout:Reading state information... 2024-03-07T02:38:05.267 INFO:teuthology.orchestra.run.smithi148.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:38:05.267 INFO:teuthology.orchestra.run.smithi148.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:38:05.267 INFO:teuthology.orchestra.run.smithi148.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:38:05.271 INFO:teuthology.orchestra.run.smithi148.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout: python2.7-minimal 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout:The following additional packages will be installed: 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout: linux-generic linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout: linux-headers-generic linux-image-5.15.0-97-generic 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout: linux-modules-5.15.0-97-generic linux-modules-extra-5.15.0-97-generic 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout:Suggested packages: 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout:Recommended packages: 2024-03-07T02:38:05.272 INFO:teuthology.orchestra.run.smithi148.stdout: thermald 2024-03-07T02:38:05.310 INFO:teuthology.orchestra.run.smithi148.stdout:The following NEW packages will be installed: 2024-03-07T02:38:05.310 INFO:teuthology.orchestra.run.smithi148.stdout: linux-headers-5.15.0-97 linux-headers-5.15.0-97-generic 2024-03-07T02:38:05.310 INFO:teuthology.orchestra.run.smithi148.stdout: linux-image-5.15.0-97-generic linux-modules-5.15.0-97-generic 2024-03-07T02:38:05.311 INFO:teuthology.orchestra.run.smithi148.stdout: linux-modules-extra-5.15.0-97-generic 2024-03-07T02:38:05.311 INFO:teuthology.orchestra.run.smithi148.stdout:The following packages will be upgraded: 2024-03-07T02:38:05.312 INFO:teuthology.orchestra.run.smithi148.stdout: linux-generic linux-headers-generic linux-image-generic 2024-03-07T02:38:05.428 INFO:teuthology.orchestra.run.smithi148.stdout:3 upgraded, 5 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:38:05.429 INFO:teuthology.orchestra.run.smithi148.stdout:Need to get 113 MB of archives. 2024-03-07T02:38:05.429 INFO:teuthology.orchestra.run.smithi148.stdout:After this operation, 582 MB of additional disk space will be used. 2024-03-07T02:38:05.429 INFO:teuthology.orchestra.run.smithi148.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-97-generic amd64 5.15.0-97.107 [22.6 MB] 2024-03-07T02:38:06.953 INFO:teuthology.orchestra.run.smithi148.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-97-generic amd64 5.15.0-97.107 [11.5 MB] 2024-03-07T02:38:07.404 INFO:teuthology.orchestra.run.smithi148.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-97-generic amd64 5.15.0-97.107 [63.8 MB] 2024-03-07T02:38:07.578 INFO:teuthology.orchestra.run.smithi195.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-97-generic amd64 5.15.0-97.107 [11.5 MB] 2024-03-07T02:38:08.294 INFO:teuthology.orchestra.run.smithi195.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-97-generic amd64 5.15.0-97.107 [63.8 MB] 2024-03-07T02:38:09.848 INFO:teuthology.orchestra.run.smithi148.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.97.92 [1,696 B] 2024-03-07T02:38:09.848 INFO:teuthology.orchestra.run.smithi148.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.97.92 [2,494 B] 2024-03-07T02:38:09.849 INFO:teuthology.orchestra.run.smithi148.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97 all 5.15.0-97.107 [12.3 MB] 2024-03-07T02:38:10.240 INFO:teuthology.orchestra.run.smithi148.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97-generic amd64 5.15.0-97.107 [2,892 kB] 2024-03-07T02:38:10.318 INFO:teuthology.orchestra.run.smithi148.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.97.92 [2,342 B] 2024-03-07T02:38:10.700 INFO:teuthology.orchestra.run.smithi148.stdout:Fetched 113 MB in 5s (22.6 MB/s) 2024-03-07T02:38:10.879 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package linux-modules-5.15.0-97-generic. 2024-03-07T02:38:10.931 INFO:teuthology.orchestra.run.smithi195.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.97.92 [1,696 B] 2024-03-07T02:38:10.931 INFO:teuthology.orchestra.run.smithi195.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.97.92 [2,494 B] 2024-03-07T02:38:10.932 INFO:teuthology.orchestra.run.smithi195.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97 all 5.15.0-97.107 [12.3 MB] 2024-03-07T02:38:11.292 INFO:teuthology.orchestra.run.smithi195.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-97-generic amd64 5.15.0-97.107 [2,892 kB] 2024-03-07T02:38:11.365 INFO:teuthology.orchestra.run.smithi195.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.97.92 [2,342 B] 2024-03-07T02:38:11.789 INFO:teuthology.orchestra.run.smithi195.stdout:Fetched 113 MB in 6s (17.6 MB/s) 2024-03-07T02:38:11.966 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package linux-modules-5.15.0-97-generic. 2024-03-07T02:38:12.839 INFO:teuthology.orchestra.run.smithi148.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-03-07T02:38:12.843 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../0-linux-modules-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:12.910 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:14.028 INFO:teuthology.orchestra.run.smithi195.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-03-07T02:38:14.035 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../0-linux-modules-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:14.105 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:14.904 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package linux-image-5.15.0-97-generic. 2024-03-07T02:38:14.926 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../1-linux-image-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:15.024 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:15.432 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-97-generic. 2024-03-07T02:38:15.453 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:15.499 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:16.173 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package linux-image-5.15.0-97-generic. 2024-03-07T02:38:16.195 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../1-linux-image-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:16.289 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:16.709 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-97-generic. 2024-03-07T02:38:16.731 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:16.768 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:21.781 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../3-linux-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:38:21.924 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:38:22.232 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:38:22.328 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-image-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:38:22.623 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package linux-headers-5.15.0-97. 2024-03-07T02:38:22.645 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../5-linux-headers-5.15.0-97_5.15.0-97.107_all.deb ... 2024-03-07T02:38:22.689 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:38:23.601 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../3-linux-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:38:23.728 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:38:24.019 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:38:24.115 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-image-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:38:24.393 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package linux-headers-5.15.0-97. 2024-03-07T02:38:24.415 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../5-linux-headers-5.15.0-97_5.15.0-97.107_all.deb ... 2024-03-07T02:38:24.459 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:38:27.995 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package linux-headers-5.15.0-97-generic. 2024-03-07T02:38:28.018 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../6-linux-headers-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:28.062 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:29.812 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package linux-headers-5.15.0-97-generic. 2024-03-07T02:38:29.851 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../6-linux-headers-5.15.0-97-generic_5.15.0-97.107_amd64.deb ... 2024-03-07T02:38:29.887 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:30.890 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:38:31.014 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking linux-headers-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:38:31.355 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:38:31.477 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:31.612 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-headers-generic (5.15.0.97.92) ... 2024-03-07T02:38:31.740 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:33.035 INFO:teuthology.orchestra.run.smithi148.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:38:33.035 INFO:teuthology.orchestra.run.smithi148.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-03-07T02:38:33.035 INFO:teuthology.orchestra.run.smithi148.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:38:33.036 INFO:teuthology.orchestra.run.smithi148.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:33.200 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:33.485 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.97.92_amd64.deb ... 2024-03-07T02:38:33.607 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking linux-headers-generic (5.15.0.97.92) over (5.15.0.56.54) ... 2024-03-07T02:38:33.949 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-headers-5.15.0-97 (5.15.0-97.107) ... 2024-03-07T02:38:34.070 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-headers-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:34.188 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-headers-generic (5.15.0.97.92) ... 2024-03-07T02:38:34.336 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:34.585 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:35.668 INFO:teuthology.orchestra.run.smithi195.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:38:35.668 INFO:teuthology.orchestra.run.smithi195.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-03-07T02:38:35.668 INFO:teuthology.orchestra.run.smithi195.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:38:35.669 INFO:teuthology.orchestra.run.smithi195.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:35.818 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-modules-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:35.995 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-image-generic (5.15.0.97.92) ... 2024-03-07T02:38:36.121 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up linux-generic (5.15.0.97.92) ... 2024-03-07T02:38:36.247 INFO:teuthology.orchestra.run.smithi148.stdout:Processing triggers for linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:36.314 INFO:teuthology.orchestra.run.smithi148.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-03-07T02:38:36.314 INFO:teuthology.orchestra.run.smithi148.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:37.223 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-modules-extra-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:39.038 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-image-generic (5.15.0.97.92) ... 2024-03-07T02:38:39.156 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up linux-generic (5.15.0.97.92) ... 2024-03-07T02:38:39.282 INFO:teuthology.orchestra.run.smithi195.stdout:Processing triggers for linux-image-5.15.0-97-generic (5.15.0-97.107) ... 2024-03-07T02:38:39.345 INFO:teuthology.orchestra.run.smithi195.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-03-07T02:38:39.346 INFO:teuthology.orchestra.run.smithi195.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:50.305 INFO:teuthology.orchestra.run.smithi148.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-03-07T02:38:50.305 INFO:teuthology.orchestra.run.smithi148.stdout:Sourcing file `/etc/default/grub' 2024-03-07T02:38:50.327 INFO:teuthology.orchestra.run.smithi148.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:38:50.359 INFO:teuthology.orchestra.run.smithi148.stdout:Generating grub configuration file ... 2024-03-07T02:38:50.615 INFO:teuthology.orchestra.run.smithi148.stdout:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:38:50.632 INFO:teuthology.orchestra.run.smithi148.stdout:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:50.917 INFO:teuthology.orchestra.run.smithi148.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:38:50.923 INFO:teuthology.orchestra.run.smithi148.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:38:50.993 INFO:teuthology.orchestra.run.smithi148.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:38:50.998 INFO:teuthology.orchestra.run.smithi148.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:38:51.152 INFO:teuthology.orchestra.run.smithi148.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:38:51.152 INFO:teuthology.orchestra.run.smithi148.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:38:51.152 INFO:teuthology.orchestra.run.smithi148.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:38:51.194 INFO:teuthology.orchestra.run.smithi148.stdout:done 2024-03-07T02:38:51.648 DEBUG:teuthology.orchestra.run.smithi148:> dpkg -s linux-image-generic 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Package: linux-image-generic 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Status: install ok installed 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Priority: optional 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Section: kernel 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Installed-Size: 21 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Architecture: amd64 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Source: linux-meta 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Version: 5.15.0.97.92 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:38:51.672 INFO:teuthology.orchestra.run.smithi148.stdout:Recommends: thermald 2024-03-07T02:38:51.673 INFO:teuthology.orchestra.run.smithi148.stdout:Description: Generic Linux kernel image 2024-03-07T02:38:51.673 INFO:teuthology.orchestra.run.smithi148.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:38:51.673 INFO:teuthology.orchestra.run.smithi148.stdout: available. 2024-03-07T02:38:51.673 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-97-generic 2024-03-07T02:38:51.673 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-07T02:38:51.673 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-07T02:38:51.673 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-03-07T02:38:51.673 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi148.front.sepia.ceph.com, path=None, version=distro) 2024-03-07T02:38:51.673 DEBUG:teuthology.orchestra.run.smithi148:> sudo apt-get clean 2024-03-07T02:38:51.852 DEBUG:teuthology.orchestra.run.smithi148:> sudo apt-get update 2024-03-07T02:38:51.973 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:38:51.973 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:38:51.986 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:38:52.006 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:38:53.237 INFO:teuthology.orchestra.run.smithi148.stdout:Reading package lists... 2024-03-07T02:38:53.249 INFO:teuthology.orchestra.run.smithi195.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-03-07T02:38:53.249 INFO:teuthology.orchestra.run.smithi195.stdout:Sourcing file `/etc/default/grub' 2024-03-07T02:38:53.254 DEBUG:teuthology.orchestra.run.smithi148:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:38:53.258 INFO:teuthology.orchestra.run.smithi195.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:38:53.274 INFO:teuthology.orchestra.run.smithi195.stdout:Generating grub configuration file ... 2024-03-07T02:38:53.313 INFO:teuthology.orchestra.run.smithi148.stdout:Reading package lists... 2024-03-07T02:38:53.503 INFO:teuthology.orchestra.run.smithi148.stdout:Building dependency tree... 2024-03-07T02:38:53.504 INFO:teuthology.orchestra.run.smithi148.stdout:Reading state information... 2024-03-07T02:38:53.560 INFO:teuthology.orchestra.run.smithi195.stdout:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:38:53.583 INFO:teuthology.orchestra.run.smithi195.stdout:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:53.676 INFO:teuthology.orchestra.run.smithi148.stdout:linux-image-generic is already the newest version (5.15.0.97.92). 2024-03-07T02:38:53.676 INFO:teuthology.orchestra.run.smithi148.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:38:53.676 INFO:teuthology.orchestra.run.smithi148.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:38:53.676 INFO:teuthology.orchestra.run.smithi148.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:38:53.677 INFO:teuthology.orchestra.run.smithi148.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:38:53.677 INFO:teuthology.orchestra.run.smithi148.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:38:53.677 INFO:teuthology.orchestra.run.smithi148.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:38:53.677 INFO:teuthology.orchestra.run.smithi148.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:38:53.717 INFO:teuthology.orchestra.run.smithi148.stdout:0 upgraded, 0 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:38:53.719 DEBUG:teuthology.orchestra.run.smithi148:> dpkg -s linux-image-generic 2024-03-07T02:38:53.734 INFO:teuthology.orchestra.run.smithi148.stdout:Package: linux-image-generic 2024-03-07T02:38:53.734 INFO:teuthology.orchestra.run.smithi148.stdout:Status: install ok installed 2024-03-07T02:38:53.734 INFO:teuthology.orchestra.run.smithi148.stdout:Priority: optional 2024-03-07T02:38:53.734 INFO:teuthology.orchestra.run.smithi148.stdout:Section: kernel 2024-03-07T02:38:53.734 INFO:teuthology.orchestra.run.smithi148.stdout:Installed-Size: 21 2024-03-07T02:38:53.734 INFO:teuthology.orchestra.run.smithi148.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:38:53.734 INFO:teuthology.orchestra.run.smithi148.stdout:Architecture: amd64 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout:Source: linux-meta 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout:Version: 5.15.0.97.92 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout:Recommends: thermald 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout:Description: Generic Linux kernel image 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:38:53.735 INFO:teuthology.orchestra.run.smithi148.stdout: available. 2024-03-07T02:38:53.735 DEBUG:teuthology.orchestra.run.smithi148:> mktemp 2024-03-07T02:38:53.783 INFO:teuthology.orchestra.run.smithi148.stdout:/tmp/tmp.t5IJEMWEh3 2024-03-07T02:38:53.783 DEBUG:teuthology.orchestra.run.smithi148:> sudo cp /boot/grub/grub.cfg /tmp/tmp.t5IJEMWEh3 2024-03-07T02:38:53.839 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod 0666 /tmp/tmp.t5IJEMWEh3 2024-03-07T02:38:53.892 INFO:teuthology.orchestra.run.smithi195.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:38:53.899 INFO:teuthology.orchestra.run.smithi195.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:38:53.956 DEBUG:teuthology.orchestra.remote:smithi148:/tmp/tmp.t5IJEMWEh3 is 10KB 2024-03-07T02:38:53.967 INFO:teuthology.orchestra.run.smithi195.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:38:53.968 DEBUG:teuthology.orchestra.run.smithi148:> rm -fr /tmp/tmp.t5IJEMWEh3 2024-03-07T02:38:53.972 INFO:teuthology.orchestra.run.smithi195.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:38:53.973 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-03-07T02:38:54.027 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:38:54.027 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-03-07T02:38:54.027 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-03-07T02:38:54.095 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-97-generic 2024-03-07T02:38:54.095 DEBUG:teuthology.orchestra.run.smithi148:> sudo update-grub 2024-03-07T02:38:54.142 INFO:teuthology.orchestra.run.smithi195.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:38:54.142 INFO:teuthology.orchestra.run.smithi195.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:38:54.143 INFO:teuthology.orchestra.run.smithi195.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:38:54.190 INFO:teuthology.orchestra.run.smithi195.stdout:done 2024-03-07T02:38:54.655 DEBUG:teuthology.orchestra.run.smithi195:> dpkg -s linux-image-generic 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Package: linux-image-generic 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Status: install ok installed 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Priority: optional 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Section: kernel 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Installed-Size: 21 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Architecture: amd64 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Source: linux-meta 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Version: 5.15.0.97.92 2024-03-07T02:38:54.681 INFO:teuthology.orchestra.run.smithi195.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:38:54.682 INFO:teuthology.orchestra.run.smithi195.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:38:54.682 INFO:teuthology.orchestra.run.smithi195.stdout:Recommends: thermald 2024-03-07T02:38:54.682 INFO:teuthology.orchestra.run.smithi195.stdout:Description: Generic Linux kernel image 2024-03-07T02:38:54.682 INFO:teuthology.orchestra.run.smithi195.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:38:54.682 INFO:teuthology.orchestra.run.smithi195.stdout: available. 2024-03-07T02:38:54.682 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-97-generic 2024-03-07T02:38:54.682 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-07T02:38:54.682 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-07T02:38:54.682 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-03-07T02:38:54.682 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi195.front.sepia.ceph.com, path=None, version=distro) 2024-03-07T02:38:54.682 DEBUG:teuthology.orchestra.run.smithi195:> sudo apt-get clean 2024-03-07T02:38:54.857 DEBUG:teuthology.orchestra.run.smithi195:> sudo apt-get update 2024-03-07T02:38:55.125 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:38:55.125 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:38:55.180 INFO:teuthology.orchestra.run.smithi148.stderr:Sourcing file `/etc/default/grub' 2024-03-07T02:38:55.181 INFO:teuthology.orchestra.run.smithi148.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:38:55.183 INFO:teuthology.orchestra.run.smithi148.stderr:Generating grub configuration file ... 2024-03-07T02:38:55.186 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:38:55.267 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:38:55.409 INFO:teuthology.orchestra.run.smithi148.stderr:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:38:55.415 INFO:teuthology.orchestra.run.smithi148.stderr:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:55.658 INFO:teuthology.orchestra.run.smithi148.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:38:55.664 INFO:teuthology.orchestra.run.smithi148.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:38:55.704 INFO:teuthology.orchestra.run.smithi148.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:38:55.709 INFO:teuthology.orchestra.run.smithi148.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:38:55.850 INFO:teuthology.orchestra.run.smithi148.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:38:55.850 INFO:teuthology.orchestra.run.smithi148.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:38:55.850 INFO:teuthology.orchestra.run.smithi148.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:38:55.880 INFO:teuthology.orchestra.run.smithi148.stderr:done 2024-03-07T02:38:55.889 DEBUG:teuthology.orchestra.run.smithi148:> sudo shutdown -r now 2024-03-07T02:38:56.512 INFO:teuthology.orchestra.run.smithi195.stdout:Reading package lists... 2024-03-07T02:38:56.530 DEBUG:teuthology.orchestra.run.smithi195:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-03-07T02:38:56.582 INFO:teuthology.orchestra.run.smithi195.stdout:Reading package lists... 2024-03-07T02:38:56.755 INFO:teuthology.orchestra.run.smithi195.stdout:Building dependency tree... 2024-03-07T02:38:56.755 INFO:teuthology.orchestra.run.smithi195.stdout:Reading state information... 2024-03-07T02:38:56.906 INFO:teuthology.orchestra.run.smithi195.stdout:linux-image-generic is already the newest version (5.15.0.97.92). 2024-03-07T02:38:56.907 INFO:teuthology.orchestra.run.smithi195.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:38:56.907 INFO:teuthology.orchestra.run.smithi195.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:38:56.907 INFO:teuthology.orchestra.run.smithi195.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:38:56.907 INFO:teuthology.orchestra.run.smithi195.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:38:56.907 INFO:teuthology.orchestra.run.smithi195.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:38:56.907 INFO:teuthology.orchestra.run.smithi195.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:38:56.907 INFO:teuthology.orchestra.run.smithi195.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:38:56.944 INFO:teuthology.orchestra.run.smithi195.stdout:0 upgraded, 0 newly installed, 0 to remove and 288 not upgraded. 2024-03-07T02:38:56.946 DEBUG:teuthology.orchestra.run.smithi195:> dpkg -s linux-image-generic 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Package: linux-image-generic 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Status: install ok installed 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Priority: optional 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Section: kernel 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Installed-Size: 21 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Maintainer: Ubuntu Kernel Team 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Architecture: amd64 2024-03-07T02:38:57.003 INFO:teuthology.orchestra.run.smithi195.stdout:Source: linux-meta 2024-03-07T02:38:57.004 INFO:teuthology.orchestra.run.smithi195.stdout:Version: 5.15.0.97.92 2024-03-07T02:38:57.004 INFO:teuthology.orchestra.run.smithi195.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.2), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-97), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.2) 2024-03-07T02:38:57.004 INFO:teuthology.orchestra.run.smithi195.stdout:Depends: linux-image-5.15.0-97-generic, linux-modules-extra-5.15.0-97-generic, linux-firmware, intel-microcode, amd64-microcode 2024-03-07T02:38:57.004 INFO:teuthology.orchestra.run.smithi195.stdout:Recommends: thermald 2024-03-07T02:38:57.004 INFO:teuthology.orchestra.run.smithi195.stdout:Description: Generic Linux kernel image 2024-03-07T02:38:57.004 INFO:teuthology.orchestra.run.smithi195.stdout: This package will always depend on the latest generic kernel image 2024-03-07T02:38:57.004 INFO:teuthology.orchestra.run.smithi195.stdout: available. 2024-03-07T02:38:57.004 DEBUG:teuthology.orchestra.run.smithi195:> mktemp 2024-03-07T02:38:57.047 INFO:teuthology.orchestra.run.smithi195.stdout:/tmp/tmp.CQtAbLHy99 2024-03-07T02:38:57.048 DEBUG:teuthology.orchestra.run.smithi195:> sudo cp /boot/grub/grub.cfg /tmp/tmp.CQtAbLHy99 2024-03-07T02:38:57.103 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 0666 /tmp/tmp.CQtAbLHy99 2024-03-07T02:38:57.219 DEBUG:teuthology.orchestra.remote:smithi195:/tmp/tmp.CQtAbLHy99 is 10KB 2024-03-07T02:38:57.231 DEBUG:teuthology.orchestra.run.smithi195:> rm -fr /tmp/tmp.CQtAbLHy99 2024-03-07T02:38:57.236 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-03-07T02:38:57.291 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T02:38:57.291 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-03-07T02:38:57.291 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-03-07T02:38:57.364 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-97-generic 2024-03-07T02:38:57.364 DEBUG:teuthology.orchestra.run.smithi195:> sudo update-grub 2024-03-07T02:38:58.516 INFO:teuthology.orchestra.run.smithi195.stderr:Sourcing file `/etc/default/grub' 2024-03-07T02:38:58.517 INFO:teuthology.orchestra.run.smithi195.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-03-07T02:38:58.519 INFO:teuthology.orchestra.run.smithi195.stderr:Generating grub configuration file ... 2024-03-07T02:38:58.703 INFO:teuthology.orchestra.run.smithi195.stderr:Found linux image: /boot/vmlinuz-5.15.0-97-generic 2024-03-07T02:38:58.708 INFO:teuthology.orchestra.run.smithi195.stderr:Found initrd image: /boot/initrd.img-5.15.0-97-generic 2024-03-07T02:38:58.960 INFO:teuthology.orchestra.run.smithi195.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-03-07T02:38:58.965 INFO:teuthology.orchestra.run.smithi195.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-03-07T02:38:59.002 INFO:teuthology.orchestra.run.smithi195.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-03-07T02:38:59.006 INFO:teuthology.orchestra.run.smithi195.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-03-07T02:38:59.117 INFO:teuthology.orchestra.run.smithi195.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-03-07T02:38:59.118 INFO:teuthology.orchestra.run.smithi195.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-03-07T02:38:59.118 INFO:teuthology.orchestra.run.smithi195.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-03-07T02:38:59.128 INFO:teuthology.orchestra.run.smithi195.stderr:done 2024-03-07T02:38:59.129 DEBUG:teuthology.orchestra.run.smithi195:> sudo shutdown -r now 2024-03-07T02:39:25.956 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-03-07T02:39:25.956 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-03-07T02:39:25.957 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:39:29.132 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-03-07T02:39:29.132 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:39:29.133 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:39:47.519 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.195 2024-03-07T02:39:56.531 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:39:56.532 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:39:59.612 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.195 2024-03-07T02:40:00.572 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.148 2024-03-07T02:40:09.580 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-03-07T02:40:09.581 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:40:11.615 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:40:11.616 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:40:14.691 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.195 2024-03-07T02:40:29.707 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:40:29.708 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:40:32.764 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.195 2024-03-07T02:40:41.021 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.148 2024-03-07T02:40:50.772 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:40:50.773 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:40:51.326 DEBUG:teuthology.orchestra.run.smithi195:> true 2024-03-07T02:40:52.022 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:40:52.022 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-97-generic"... 2024-03-07T02:40:52.022 DEBUG:teuthology.orchestra.run.smithi195:> uname -r 2024-03-07T02:40:52.071 INFO:teuthology.orchestra.run.smithi195.stdout:5.15.0-97-generic 2024-03-07T02:40:52.071 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-97-generic vs 5.15.0-97-generic 2024-03-07T02:40:52.071 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-07T02:40:52.071 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-03-07T02:40:53.022 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-03-07T02:40:53.022 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:40:53.073 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-03-07T02:40:53.073 DEBUG:teuthology.orchestra.run.smithi195:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-07T02:40:53.423 INFO:teuthology.orchestra.run.smithi195.stdout:ttyS1 2024-03-07T02:40:53.444 DEBUG:teuthology.parallel:result is None 2024-03-07T02:40:53.474 DEBUG:teuthology.orchestra.run.smithi148:> true 2024-03-07T02:40:53.993 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-03-07T02:40:53.993 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-97-generic"... 2024-03-07T02:40:53.993 DEBUG:teuthology.orchestra.run.smithi148:> uname -r 2024-03-07T02:40:54.041 INFO:teuthology.orchestra.run.smithi148.stdout:5.15.0-97-generic 2024-03-07T02:40:54.041 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-97-generic vs 5.15.0-97-generic 2024-03-07T02:40:54.041 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-07T02:40:54.041 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-03-07T02:40:55.042 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-03-07T02:40:55.042 DEBUG:teuthology.orchestra.run.smithi148:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-07T02:40:55.412 INFO:teuthology.orchestra.run.smithi148.stdout:ttyS1 2024-03-07T02:40:55.444 DEBUG:teuthology.parallel:result is None 2024-03-07T02:40:55.444 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-07T02:40:55.451 INFO:teuthology.task.internal:Creating test directory... 2024-03-07T02:40:55.452 DEBUG:teuthology.orchestra.run.smithi148:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-07T02:40:55.459 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-07T02:40:55.464 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-07T02:40:55.474 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-07T02:40:55.482 INFO:teuthology.task.internal:Creating archive directory... 2024-03-07T02:40:55.482 DEBUG:teuthology.orchestra.run.smithi148:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-07T02:40:55.508 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-07T02:40:55.617 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-07T02:40:55.631 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-07T02:40:55.635 DEBUG:teuthology.orchestra.run.smithi148:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-07T02:40:55.646 DEBUG:teuthology.orchestra.run.smithi195:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-07T02:40:55.683 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:55.684 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:55.684 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:55.684 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-07T02:40:55.693 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-07T02:40:55.700 INFO:teuthology.task.internal:Configuring sudo... 2024-03-07T02:40:55.700 DEBUG:teuthology.orchestra.run.smithi148:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-07T02:40:55.731 DEBUG:teuthology.orchestra.run.smithi195:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-07T02:40:55.745 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-07T02:40:55.759 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-07T02:40:55.759 DEBUG:teuthology.orchestra.run.smithi148:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-07T02:40:55.789 DEBUG:teuthology.orchestra.run.smithi195:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-07T02:40:55.802 DEBUG:teuthology.orchestra.run.smithi148:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-07T02:40:55.839 DEBUG:teuthology.orchestra.run.smithi148:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-07T02:40:55.888 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:40:55.888 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-07T02:40:55.960 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-07T02:40:55.982 DEBUG:teuthology.orchestra.run.smithi195:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-07T02:40:56.030 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T02:40:56.030 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-07T02:40:56.131 DEBUG:teuthology.orchestra.run.smithi148:> sudo service rsyslog restart 2024-03-07T02:40:56.134 DEBUG:teuthology.orchestra.run.smithi195:> sudo service rsyslog restart 2024-03-07T02:40:56.205 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-07T02:40:56.213 INFO:teuthology.task.internal:Starting timer... 2024-03-07T02:40:56.213 INFO:teuthology.run_tasks:Running task pcp... 2024-03-07T02:40:56.250 INFO:teuthology.run_tasks:Running task selinux... 2024-03-07T02:40:56.266 DEBUG:teuthology.task.selinux:Excluding smithi148: OS 'ubuntu' does not support SELinux 2024-03-07T02:40:56.266 DEBUG:teuthology.task.selinux:Excluding smithi195: OS 'ubuntu' does not support SELinux 2024-03-07T02:40:56.269 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-07T02:40:56.269 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-03-07T02:40:56.269 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-07T02:40:56.269 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-07T02:40:56.298 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-07T02:40:56.386 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-07T02:40:56.387 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi148.front.sepia.ceph.com,smithi195.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-07T02:49:54.641 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi148.front.sepia.ceph.com'), Remote(name='ubuntu@smithi195.front.sepia.ceph.com')] 2024-03-07T02:49:54.642 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-03-07T02:49:54.643 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi148.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:49:54.725 DEBUG:teuthology.orchestra.run.smithi148:> true 2024-03-07T02:49:54.804 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi148.front.sepia.ceph.com' 2024-03-07T02:49:54.805 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:49:54.805 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi195.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-07T02:49:54.881 DEBUG:teuthology.orchestra.run.smithi195:> true 2024-03-07T02:49:54.956 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi195.front.sepia.ceph.com' 2024-03-07T02:49:54.956 INFO:teuthology.run_tasks:Running task clock... 2024-03-07T02:49:54.968 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-07T02:49:54.968 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-07T02:49:54.968 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T02:49:54.971 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-07T02:49:54.971 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Command line: ntpd -gq 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: ---------------------------------------------------- 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: ntp-4 is maintained by Network Time Foundation, 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: corporation. Support and training for ntp-4 are 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: available at https://www.nwtime.org/support 2024-03-07T02:49:54.997 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: ---------------------------------------------------- 2024-03-07T02:49:54.999 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: proto: precision = 0.057 usec (-24) 2024-03-07T02:49:54.999 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: basedate set to 2022-02-04 2024-03-07T02:49:54.999 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: gps base set to 2022-02-06 (week 2196) 2024-03-07T02:49:54.999 INFO:teuthology.orchestra.run.smithi148.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:54.999 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:54.999 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:55.000 INFO:teuthology.orchestra.run.smithi148.stderr:restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:55.001 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Listen and drop on 0 v6wildcard [::]:123 2024-03-07T02:49:55.001 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-03-07T02:49:55.001 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Listen normally on 2 lo 127.0.0.1:123 2024-03-07T02:49:55.001 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Listen normally on 3 enp3s0f1 172.21.15.148:123 2024-03-07T02:49:55.001 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Listen normally on 4 lo [::1]:123 2024-03-07T02:49:55.001 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:7257%5]:123 2024-03-07T02:49:55.001 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:49:54 ntpd[17160]: Listening on routing socket on fd #22 for interface updates 2024-03-07T02:49:55.022 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-03-07T02:49:55.022 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Command line: ntpd -gq 2024-03-07T02:49:55.022 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: ---------------------------------------------------- 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: ntp-4 is maintained by Network Time Foundation, 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: corporation. Support and training for ntp-4 are 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: available at https://www.nwtime.org/support 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: ---------------------------------------------------- 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: proto: precision = 0.045 usec (-24) 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: basedate set to 2022-02-04 2024-03-07T02:49:55.023 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: gps base set to 2022-02-06 (week 2196) 2024-03-07T02:49:55.024 INFO:teuthology.orchestra.run.smithi195.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:55.024 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-03-07T02:49:55.024 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:55.024 INFO:teuthology.orchestra.run.smithi195.stderr:restrict ::: KOD does nothing without LIMITED. 2024-03-07T02:49:55.025 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Listen and drop on 0 v6wildcard [::]:123 2024-03-07T02:49:55.025 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-03-07T02:49:55.025 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Listen normally on 2 lo 127.0.0.1:123 2024-03-07T02:49:55.025 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Listen normally on 3 enp3s0f1 172.21.15.195:123 2024-03-07T02:49:55.026 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Listen normally on 4 lo [::1]:123 2024-03-07T02:49:55.026 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe88:8081%5]:123 2024-03-07T02:49:55.026 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:49:55 ntpd[17156]: Listening on routing socket on fd #22 for interface updates 2024-03-07T02:49:55.997 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:55 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:55.997 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:55 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:56.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:56 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:56.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:56 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:56.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:56 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:56.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:56 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:56.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:56 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:56.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:56 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:56.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:56 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:56.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:56 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:56.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:56 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:56.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:56 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:56.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:56 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:56.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:56 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:57.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:57 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:57.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:57 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:57.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:57 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:57.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:57 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:57.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:57 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:57.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:57 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:58.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:58 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:58.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:58 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:58.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:58 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:58.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:58 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:58.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:58 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:58.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:58 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:58.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:58.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:58.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:58.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:58.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:58.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:59.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:49:58 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:49:59.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:59 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:59.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:59 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:59.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:59 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:49:59.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:59 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:49:59.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:59 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:49:59.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:49:59 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:00.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:00.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:00 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:00.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:01.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:00 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:01.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:01 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:01.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:01 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:01.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:01 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:01.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:01 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:01.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:01 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:01.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:01 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:02.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:02.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:02.023 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:02.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:02.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:02.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:02.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:02.024 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:02.024 INFO:teuthology.orchestra.run.smithi195.stdout: 7 Mar 02:50:02 ntpd[17156]: ntpd: time slew +0.000658 s 2024-03-07T02:50:02.024 INFO:teuthology.orchestra.run.smithi195.stdout:ntpd: time slew +0.000658s 2024-03-07T02:50:02.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-03-07T02:50:02.025 INFO:teuthology.orchestra.run.smithi195.stderr: 7 Mar 02:50:02 ntpd[17156]: can't open /var/log/ntpstats/loopstats.20240307: Permission denied 2024-03-07T02:50:02.071 INFO:teuthology.orchestra.run.smithi195.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T02:50:02.071 INFO:teuthology.orchestra.run.smithi195.stdout:============================================================================== 2024-03-07T02:50:02.071 INFO:teuthology.orchestra.run.smithi195.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:02.071 INFO:teuthology.orchestra.run.smithi195.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:02.071 INFO:teuthology.orchestra.run.smithi195.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:02.071 INFO:teuthology.orchestra.run.smithi195.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:02.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:02.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:02.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:02.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:02.998 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:02.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:02.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-03-07T02:50:02.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: can't open /var/log/ntpstats/rawstats.20240307: Permission denied 2024-03-07T02:50:02.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-03-07T02:50:02.999 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: can't open /var/log/ntpstats/peerstats.20240307: Permission denied 2024-03-07T02:50:02.999 INFO:teuthology.orchestra.run.smithi148.stdout: 7 Mar 02:50:02 ntpd[17160]: ntpd: time slew -0.000025 s 2024-03-07T02:50:02.999 INFO:teuthology.orchestra.run.smithi148.stdout:ntpd: time slew -0.000025s 2024-03-07T02:50:03.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-03-07T02:50:03.000 INFO:teuthology.orchestra.run.smithi148.stderr: 7 Mar 02:50:02 ntpd[17160]: can't open /var/log/ntpstats/loopstats.20240307: Permission denied 2024-03-07T02:50:03.268 INFO:teuthology.orchestra.run.smithi148.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T02:50:03.268 INFO:teuthology.orchestra.run.smithi148.stdout:============================================================================== 2024-03-07T02:50:03.268 INFO:teuthology.orchestra.run.smithi148.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:03.268 INFO:teuthology.orchestra.run.smithi148.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:03.268 INFO:teuthology.orchestra.run.smithi148.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:03.268 INFO:teuthology.orchestra.run.smithi148.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-03-07T02:50:03.269 INFO:teuthology.run_tasks:Running task install... 2024-03-07T02:50:03.278 DEBUG:teuthology.task.install:project ceph 2024-03-07T02:50:03.278 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71'}} 2024-03-07T02:50:03.278 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71'} 2024-03-07T02:50:03.278 INFO:teuthology.task.install:Using flavor: default 2024-03-07T02:50:03.284 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-07T02:50:03.284 INFO:teuthology.task.install:extra packages: [] 2024-03-07T02:50:03.284 DEBUG:teuthology.orchestra.run.smithi148:> sudo apt-key list | grep Ceph 2024-03-07T02:50:03.285 DEBUG:teuthology.orchestra.run.smithi195:> sudo apt-key list | grep Ceph 2024-03-07T02:50:03.347 INFO:teuthology.orchestra.run.smithi195.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-03-07T02:50:03.373 INFO:teuthology.orchestra.run.smithi195.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-03-07T02:50:03.373 INFO:teuthology.orchestra.run.smithi195.stdout:uid [ unknown] Ceph.com (release key) 2024-03-07T02:50:03.374 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-03-07T02:50:03.374 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:50:03.374 INFO:teuthology.packaging:ref: None 2024-03-07T02:50:03.374 INFO:teuthology.packaging:tag: None 2024-03-07T02:50:03.374 INFO:teuthology.packaging:branch: reef 2024-03-07T02:50:03.374 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:50:03.374 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:50:03.379 INFO:teuthology.orchestra.run.smithi148.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-03-07T02:50:03.409 INFO:teuthology.orchestra.run.smithi148.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-03-07T02:50:03.409 INFO:teuthology.orchestra.run.smithi148.stdout:uid [ unknown] Ceph.com (release key) 2024-03-07T02:50:03.409 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-03-07T02:50:03.409 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:50:03.409 INFO:teuthology.packaging:ref: None 2024-03-07T02:50:03.409 INFO:teuthology.packaging:tag: None 2024-03-07T02:50:03.409 INFO:teuthology.packaging:branch: reef 2024-03-07T02:50:03.409 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:50:03.409 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:50:03.597 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default/ 2024-03-07T02:50:03.597 INFO:teuthology.task.install.deb:Package version is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:50:03.647 INFO:teuthology.task.install.deb:Pulling from https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default/ 2024-03-07T02:50:03.647 INFO:teuthology.task.install.deb:Package version is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:50:03.762 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T02:50:03.762 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-03-07T02:50:03.774 DEBUG:teuthology.orchestra.run.smithi195:> sudo apt-get update 2024-03-07T02:50:03.781 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:50:03.781 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-03-07T02:50:03.794 DEBUG:teuthology.orchestra.run.smithi148:> sudo apt-get update 2024-03-07T02:50:03.946 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:50:03.960 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:50:03.981 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:50:04.078 INFO:teuthology.orchestra.run.smithi195.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:50:04.088 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-03-07T02:50:04.089 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-03-07T02:50:04.089 INFO:teuthology.orchestra.run.smithi148.stdout:Ign:3 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy InRelease 2024-03-07T02:50:04.113 INFO:teuthology.orchestra.run.smithi195.stdout:Ign:5 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy InRelease 2024-03-07T02:50:04.130 INFO:teuthology.orchestra.run.smithi148.stdout:Get:4 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-03-07T02:50:04.160 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:5 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-03-07T02:50:04.171 INFO:teuthology.orchestra.run.smithi195.stdout:Get:6 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-03-07T02:50:04.175 INFO:teuthology.orchestra.run.smithi148.stdout:Ign:6 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release.gpg 2024-03-07T02:50:04.217 INFO:teuthology.orchestra.run.smithi148.stdout:Get:7 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.5 kB] 2024-03-07T02:50:04.229 INFO:teuthology.orchestra.run.smithi195.stdout:Ign:7 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy Release.gpg 2024-03-07T02:50:04.244 INFO:teuthology.orchestra.run.smithi148.stdout:Hit:8 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-03-07T02:50:04.266 INFO:teuthology.orchestra.run.smithi148.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,409 B] 2024-03-07T02:50:04.287 INFO:teuthology.orchestra.run.smithi195.stdout:Get:8 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.5 kB] 2024-03-07T02:50:04.410 INFO:teuthology.orchestra.run.smithi195.stdout:Get:9 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,409 B] 2024-03-07T02:50:04.438 INFO:teuthology.orchestra.run.smithi148.stdout:Fetched 28.5 kB in 0s (60.8 kB/s) 2024-03-07T02:50:04.491 INFO:teuthology.orchestra.run.smithi195.stdout:Fetched 28.5 kB in 1s (53.7 kB/s) 2024-03-07T02:50:05.316 INFO:teuthology.orchestra.run.smithi148.stdout:Reading package lists... 2024-03-07T02:50:05.333 DEBUG:teuthology.orchestra.run.smithi148:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.2.1-707-gffcfe7db-1jammy cephadm=18.2.1-707-gffcfe7db-1jammy ceph-mds=18.2.1-707-gffcfe7db-1jammy ceph-mgr=18.2.1-707-gffcfe7db-1jammy ceph-common=18.2.1-707-gffcfe7db-1jammy ceph-fuse=18.2.1-707-gffcfe7db-1jammy ceph-test=18.2.1-707-gffcfe7db-1jammy radosgw=18.2.1-707-gffcfe7db-1jammy python3-rados=18.2.1-707-gffcfe7db-1jammy python3-rgw=18.2.1-707-gffcfe7db-1jammy python3-cephfs=18.2.1-707-gffcfe7db-1jammy python3-rbd=18.2.1-707-gffcfe7db-1jammy libcephfs2=18.2.1-707-gffcfe7db-1jammy libcephfs-dev=18.2.1-707-gffcfe7db-1jammy librados2=18.2.1-707-gffcfe7db-1jammy librbd1=18.2.1-707-gffcfe7db-1jammy rbd-fuse=18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:50:05.393 INFO:teuthology.orchestra.run.smithi148.stdout:Reading package lists... 2024-03-07T02:50:05.444 INFO:teuthology.orchestra.run.smithi195.stdout:Reading package lists... 2024-03-07T02:50:05.460 DEBUG:teuthology.orchestra.run.smithi195:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.2.1-707-gffcfe7db-1jammy cephadm=18.2.1-707-gffcfe7db-1jammy ceph-mds=18.2.1-707-gffcfe7db-1jammy ceph-mgr=18.2.1-707-gffcfe7db-1jammy ceph-common=18.2.1-707-gffcfe7db-1jammy ceph-fuse=18.2.1-707-gffcfe7db-1jammy ceph-test=18.2.1-707-gffcfe7db-1jammy radosgw=18.2.1-707-gffcfe7db-1jammy python3-rados=18.2.1-707-gffcfe7db-1jammy python3-rgw=18.2.1-707-gffcfe7db-1jammy python3-cephfs=18.2.1-707-gffcfe7db-1jammy python3-rbd=18.2.1-707-gffcfe7db-1jammy libcephfs2=18.2.1-707-gffcfe7db-1jammy libcephfs-dev=18.2.1-707-gffcfe7db-1jammy librados2=18.2.1-707-gffcfe7db-1jammy librbd1=18.2.1-707-gffcfe7db-1jammy rbd-fuse=18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:50:05.512 INFO:teuthology.orchestra.run.smithi195.stdout:Reading package lists... 2024-03-07T02:50:05.581 INFO:teuthology.orchestra.run.smithi148.stdout:Building dependency tree... 2024-03-07T02:50:05.581 INFO:teuthology.orchestra.run.smithi148.stdout:Reading state information... 2024-03-07T02:50:05.683 INFO:teuthology.orchestra.run.smithi195.stdout:Building dependency tree... 2024-03-07T02:50:05.684 INFO:teuthology.orchestra.run.smithi195.stdout:Reading state information... 2024-03-07T02:50:05.758 INFO:teuthology.orchestra.run.smithi148.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:50:05.758 INFO:teuthology.orchestra.run.smithi148.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:50:05.758 INFO:teuthology.orchestra.run.smithi148.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:50:05.758 INFO:teuthology.orchestra.run.smithi148.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:50:05.759 INFO:teuthology.orchestra.run.smithi148.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:50:05.759 INFO:teuthology.orchestra.run.smithi148.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:50:05.759 INFO:teuthology.orchestra.run.smithi148.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:50:05.759 INFO:teuthology.orchestra.run.smithi148.stdout:The following additional packages will be installed: 2024-03-07T02:50:05.759 INFO:teuthology.orchestra.run.smithi148.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-03-07T02:50:05.760 INFO:teuthology.orchestra.run.smithi148.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-03-07T02:50:05.760 INFO:teuthology.orchestra.run.smithi148.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-03-07T02:50:05.760 INFO:teuthology.orchestra.run.smithi148.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-03-07T02:50:05.761 INFO:teuthology.orchestra.run.smithi148.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout:Suggested packages: 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-03-07T02:50:05.762 INFO:teuthology.orchestra.run.smithi148.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-03-07T02:50:05.763 INFO:teuthology.orchestra.run.smithi148.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-03-07T02:50:05.763 INFO:teuthology.orchestra.run.smithi148.stdout: python3-watchdog 2024-03-07T02:50:05.763 INFO:teuthology.orchestra.run.smithi148.stdout:Recommended packages: 2024-03-07T02:50:05.763 INFO:teuthology.orchestra.run.smithi148.stdout: btrfs-tools 2024-03-07T02:50:05.840 INFO:teuthology.orchestra.run.smithi195.stdout:The following packages were automatically installed and are no longer required: 2024-03-07T02:50:05.840 INFO:teuthology.orchestra.run.smithi195.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-03-07T02:50:05.840 INFO:teuthology.orchestra.run.smithi195.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-03-07T02:50:05.840 INFO:teuthology.orchestra.run.smithi195.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-03-07T02:50:05.840 INFO:teuthology.orchestra.run.smithi195.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-03-07T02:50:05.841 INFO:teuthology.orchestra.run.smithi195.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-03-07T02:50:05.841 INFO:teuthology.orchestra.run.smithi195.stdout:Use 'sudo apt autoremove' to remove them. 2024-03-07T02:50:05.841 INFO:teuthology.orchestra.run.smithi195.stdout:The following additional packages will be installed: 2024-03-07T02:50:05.841 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-03-07T02:50:05.842 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-03-07T02:50:05.842 INFO:teuthology.orchestra.run.smithi195.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-03-07T02:50:05.842 INFO:teuthology.orchestra.run.smithi195.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-03-07T02:50:05.842 INFO:teuthology.orchestra.run.smithi195.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-03-07T02:50:05.842 INFO:teuthology.orchestra.run.smithi195.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-03-07T02:50:05.842 INFO:teuthology.orchestra.run.smithi195.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:50:05.843 INFO:teuthology.orchestra.run.smithi195.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout:Suggested packages: 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: python3-watchdog 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout:Recommended packages: 2024-03-07T02:50:05.844 INFO:teuthology.orchestra.run.smithi195.stdout: btrfs-tools 2024-03-07T02:50:05.862 INFO:teuthology.orchestra.run.smithi148.stdout:The following NEW packages will be installed: 2024-03-07T02:50:05.862 INFO:teuthology.orchestra.run.smithi148.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-03-07T02:50:05.862 INFO:teuthology.orchestra.run.smithi148.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-03-07T02:50:05.862 INFO:teuthology.orchestra.run.smithi148.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-03-07T02:50:05.862 INFO:teuthology.orchestra.run.smithi148.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-03-07T02:50:05.862 INFO:teuthology.orchestra.run.smithi148.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-03-07T02:50:05.862 INFO:teuthology.orchestra.run.smithi148.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-03-07T02:50:05.863 INFO:teuthology.orchestra.run.smithi148.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python-babel-localedata 2024-03-07T02:50:05.863 INFO:teuthology.orchestra.run.smithi148.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-03-07T02:50:05.863 INFO:teuthology.orchestra.run.smithi148.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2024-03-07T02:50:05.863 INFO:teuthology.orchestra.run.smithi148.stdout: python3-certifi python3-cheroot python3-cherrypy3 python3-google-auth 2024-03-07T02:50:05.863 INFO:teuthology.orchestra.run.smithi148.stdout: python3-idna python3-iniconfig python3-jaraco.classes 2024-03-07T02:50:05.863 INFO:teuthology.orchestra.run.smithi148.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2024-03-07T02:50:05.863 INFO:teuthology.orchestra.run.smithi148.stdout: python3-jinja2 python3-joblib python3-kubernetes python3-logutils 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-mako python3-markupsafe python3-natsort python3-openssl 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-pyasn1 python3-pyasn1-modules python3-pygments python3-pyinotify 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-pytest python3-rados python3-rbd python3-repoze.lru python3-requests 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2024-03-07T02:50:05.864 INFO:teuthology.orchestra.run.smithi148.stdout: socat xmlstarlet 2024-03-07T02:50:05.927 INFO:teuthology.orchestra.run.smithi148.stdout:0 upgraded, 109 newly installed, 0 to remove and 275 not upgraded. 2024-03-07T02:50:05.927 INFO:teuthology.orchestra.run.smithi148.stdout:Need to get 181 MB of archives. 2024-03-07T02:50:05.927 INFO:teuthology.orchestra.run.smithi148.stdout:After this operation, 811 MB of additional disk space will be used. 2024-03-07T02:50:05.927 INFO:teuthology.orchestra.run.smithi148.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-03-07T02:50:05.940 INFO:teuthology.orchestra.run.smithi195.stdout:The following NEW packages will be installed: 2024-03-07T02:50:05.940 INFO:teuthology.orchestra.run.smithi195.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-03-07T02:50:05.940 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-03-07T02:50:05.940 INFO:teuthology.orchestra.run.smithi195.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-03-07T02:50:05.941 INFO:teuthology.orchestra.run.smithi195.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-03-07T02:50:05.941 INFO:teuthology.orchestra.run.smithi195.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-03-07T02:50:05.941 INFO:teuthology.orchestra.run.smithi195.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python-babel-localedata 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-certifi python3-cheroot python3-cherrypy3 python3-google-auth 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-idna python3-iniconfig python3-jaraco.classes 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-jinja2 python3-joblib python3-kubernetes python3-logutils 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-mako python3-markupsafe python3-natsort python3-openssl 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pyasn1 python3-pyasn1-modules python3-pygments python3-pyinotify 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-pytest python3-rados python3-rbd python3-repoze.lru python3-requests 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-03-07T02:50:05.942 INFO:teuthology.orchestra.run.smithi195.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-03-07T02:50:05.943 INFO:teuthology.orchestra.run.smithi195.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2024-03-07T02:50:05.943 INFO:teuthology.orchestra.run.smithi195.stdout: socat xmlstarlet 2024-03-07T02:50:05.994 INFO:teuthology.orchestra.run.smithi148.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-03-07T02:50:06.004 INFO:teuthology.orchestra.run.smithi195.stdout:0 upgraded, 109 newly installed, 0 to remove and 275 not upgraded. 2024-03-07T02:50:06.005 INFO:teuthology.orchestra.run.smithi195.stdout:Need to get 181 MB of archives. 2024-03-07T02:50:06.005 INFO:teuthology.orchestra.run.smithi195.stdout:After this operation, 811 MB of additional disk space will be used. 2024-03-07T02:50:06.005 INFO:teuthology.orchestra.run.smithi195.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-03-07T02:50:06.038 INFO:teuthology.orchestra.run.smithi148.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-03-07T02:50:06.041 INFO:teuthology.orchestra.run.smithi148.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-03-07T02:50:06.058 INFO:teuthology.orchestra.run.smithi148.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-03-07T02:50:06.072 INFO:teuthology.orchestra.run.smithi195.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-03-07T02:50:06.116 INFO:teuthology.orchestra.run.smithi148.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-03-07T02:50:06.120 INFO:teuthology.orchestra.run.smithi195.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-03-07T02:50:06.121 INFO:teuthology.orchestra.run.smithi148.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-03-07T02:50:06.125 INFO:teuthology.orchestra.run.smithi195.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-03-07T02:50:06.133 INFO:teuthology.orchestra.run.smithi148.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-03-07T02:50:06.137 INFO:teuthology.orchestra.run.smithi148.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-03-07T02:50:06.137 INFO:teuthology.orchestra.run.smithi148.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-03-07T02:50:06.138 INFO:teuthology.orchestra.run.smithi148.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-03-07T02:50:06.147 INFO:teuthology.orchestra.run.smithi148.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-03-07T02:50:06.148 INFO:teuthology.orchestra.run.smithi195.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-03-07T02:50:06.148 INFO:teuthology.orchestra.run.smithi148.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-03-07T02:50:06.148 INFO:teuthology.orchestra.run.smithi148.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-03-07T02:50:06.149 INFO:teuthology.orchestra.run.smithi148.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-03-07T02:50:06.167 INFO:teuthology.orchestra.run.smithi148.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-03-07T02:50:06.168 INFO:teuthology.orchestra.run.smithi148.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-03-07T02:50:06.168 INFO:teuthology.orchestra.run.smithi148.stdout:Get:18 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 18.2.1-707-gffcfe7db-1jammy [3,418 kB] 2024-03-07T02:50:06.168 INFO:teuthology.orchestra.run.smithi148.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-03-07T02:50:06.169 INFO:teuthology.orchestra.run.smithi148.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-03-07T02:50:06.169 INFO:teuthology.orchestra.run.smithi148.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-03-07T02:50:06.187 INFO:teuthology.orchestra.run.smithi148.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-03-07T02:50:06.190 INFO:teuthology.orchestra.run.smithi148.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-03-07T02:50:06.191 INFO:teuthology.orchestra.run.smithi148.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-03-07T02:50:06.191 INFO:teuthology.orchestra.run.smithi148.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-03-07T02:50:06.192 INFO:teuthology.orchestra.run.smithi148.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-03-07T02:50:06.210 INFO:teuthology.orchestra.run.smithi148.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-03-07T02:50:06.211 INFO:teuthology.orchestra.run.smithi148.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-03-07T02:50:06.211 INFO:teuthology.orchestra.run.smithi148.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-03-07T02:50:06.212 INFO:teuthology.orchestra.run.smithi195.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-03-07T02:50:06.213 INFO:teuthology.orchestra.run.smithi148.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-03-07T02:50:06.214 INFO:teuthology.orchestra.run.smithi148.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-03-07T02:50:06.217 INFO:teuthology.orchestra.run.smithi195.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-03-07T02:50:06.230 INFO:teuthology.orchestra.run.smithi195.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-03-07T02:50:06.235 INFO:teuthology.orchestra.run.smithi195.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-03-07T02:50:06.235 INFO:teuthology.orchestra.run.smithi195.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-03-07T02:50:06.236 INFO:teuthology.orchestra.run.smithi195.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-03-07T02:50:06.238 INFO:teuthology.orchestra.run.smithi148.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-03-07T02:50:06.245 INFO:teuthology.orchestra.run.smithi195.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-03-07T02:50:06.246 INFO:teuthology.orchestra.run.smithi195.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-03-07T02:50:06.246 INFO:teuthology.orchestra.run.smithi195.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-03-07T02:50:06.246 INFO:teuthology.orchestra.run.smithi148.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-03-07T02:50:06.247 INFO:teuthology.orchestra.run.smithi148.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-03-07T02:50:06.247 INFO:teuthology.orchestra.run.smithi195.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-03-07T02:50:06.248 INFO:teuthology.orchestra.run.smithi148.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-03-07T02:50:06.248 INFO:teuthology.orchestra.run.smithi148.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-03-07T02:50:06.262 INFO:teuthology.orchestra.run.smithi148.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-03-07T02:50:06.264 INFO:teuthology.orchestra.run.smithi148.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB] 2024-03-07T02:50:06.264 INFO:teuthology.orchestra.run.smithi195.stdout:Get:16 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 18.2.1-707-gffcfe7db-1jammy [3,418 kB] 2024-03-07T02:50:06.265 INFO:teuthology.orchestra.run.smithi148.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-03-07T02:50:06.267 INFO:teuthology.orchestra.run.smithi148.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-03-07T02:50:06.268 INFO:teuthology.orchestra.run.smithi148.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB] 2024-03-07T02:50:06.273 INFO:teuthology.orchestra.run.smithi195.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-03-07T02:50:06.273 INFO:teuthology.orchestra.run.smithi195.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-03-07T02:50:06.273 INFO:teuthology.orchestra.run.smithi195.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-03-07T02:50:06.274 INFO:teuthology.orchestra.run.smithi195.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-03-07T02:50:06.274 INFO:teuthology.orchestra.run.smithi195.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-03-07T02:50:06.274 INFO:teuthology.orchestra.run.smithi195.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-03-07T02:50:06.278 INFO:teuthology.orchestra.run.smithi195.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-03-07T02:50:06.278 INFO:teuthology.orchestra.run.smithi195.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-03-07T02:50:06.282 INFO:teuthology.orchestra.run.smithi148.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-03-07T02:50:06.283 INFO:teuthology.orchestra.run.smithi148.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-03-07T02:50:06.286 INFO:teuthology.orchestra.run.smithi148.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-03-07T02:50:06.306 INFO:teuthology.orchestra.run.smithi195.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-03-07T02:50:06.307 INFO:teuthology.orchestra.run.smithi195.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-03-07T02:50:06.352 INFO:teuthology.orchestra.run.smithi195.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-03-07T02:50:06.353 INFO:teuthology.orchestra.run.smithi195.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-03-07T02:50:06.353 INFO:teuthology.orchestra.run.smithi195.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-03-07T02:50:06.356 INFO:teuthology.orchestra.run.smithi195.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-03-07T02:50:06.357 INFO:teuthology.orchestra.run.smithi195.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-03-07T02:50:06.358 INFO:teuthology.orchestra.run.smithi195.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-03-07T02:50:06.359 INFO:teuthology.orchestra.run.smithi148.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-03-07T02:50:06.364 INFO:teuthology.orchestra.run.smithi148.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB] 2024-03-07T02:50:06.366 INFO:teuthology.orchestra.run.smithi148.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-03-07T02:50:06.367 INFO:teuthology.orchestra.run.smithi148.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-03-07T02:50:06.368 INFO:teuthology.orchestra.run.smithi148.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-03-07T02:50:06.369 INFO:teuthology.orchestra.run.smithi195.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-03-07T02:50:06.369 INFO:teuthology.orchestra.run.smithi148.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-03-07T02:50:06.369 INFO:teuthology.orchestra.run.smithi195.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-03-07T02:50:06.370 INFO:teuthology.orchestra.run.smithi148.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-03-07T02:50:06.370 INFO:teuthology.orchestra.run.smithi148.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-03-07T02:50:06.373 INFO:teuthology.orchestra.run.smithi148.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-03-07T02:50:06.386 INFO:teuthology.orchestra.run.smithi195.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-03-07T02:50:06.387 INFO:teuthology.orchestra.run.smithi195.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-03-07T02:50:06.395 INFO:teuthology.orchestra.run.smithi148.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-03-07T02:50:06.396 INFO:teuthology.orchestra.run.smithi148.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-03-07T02:50:06.398 INFO:teuthology.orchestra.run.smithi195.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-03-07T02:50:06.402 INFO:teuthology.orchestra.run.smithi195.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB] 2024-03-07T02:50:06.404 INFO:teuthology.orchestra.run.smithi195.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-03-07T02:50:06.406 INFO:teuthology.orchestra.run.smithi195.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-03-07T02:50:06.408 INFO:teuthology.orchestra.run.smithi195.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB] 2024-03-07T02:50:06.412 INFO:teuthology.orchestra.run.smithi195.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-03-07T02:50:06.414 INFO:teuthology.orchestra.run.smithi195.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-03-07T02:50:06.426 INFO:teuthology.orchestra.run.smithi148.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-03-07T02:50:06.429 INFO:teuthology.orchestra.run.smithi148.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-03-07T02:50:06.430 INFO:teuthology.orchestra.run.smithi195.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-03-07T02:50:06.448 INFO:teuthology.orchestra.run.smithi148.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-03-07T02:50:06.543 INFO:teuthology.orchestra.run.smithi148.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-03-07T02:50:06.543 INFO:teuthology.orchestra.run.smithi148.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-03-07T02:50:06.545 INFO:teuthology.orchestra.run.smithi148.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-03-07T02:50:06.547 INFO:teuthology.orchestra.run.smithi148.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-03-07T02:50:06.573 INFO:teuthology.orchestra.run.smithi195.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-03-07T02:50:06.575 INFO:teuthology.orchestra.run.smithi195.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB] 2024-03-07T02:50:06.577 INFO:teuthology.orchestra.run.smithi195.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-03-07T02:50:06.577 INFO:teuthology.orchestra.run.smithi195.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-03-07T02:50:06.578 INFO:teuthology.orchestra.run.smithi195.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-03-07T02:50:06.580 INFO:teuthology.orchestra.run.smithi195.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-03-07T02:50:06.580 INFO:teuthology.orchestra.run.smithi195.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-03-07T02:50:06.580 INFO:teuthology.orchestra.run.smithi195.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-03-07T02:50:06.584 INFO:teuthology.orchestra.run.smithi195.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-03-07T02:50:06.589 INFO:teuthology.orchestra.run.smithi148.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-03-07T02:50:06.596 INFO:teuthology.orchestra.run.smithi195.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-03-07T02:50:06.604 INFO:teuthology.orchestra.run.smithi195.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-03-07T02:50:06.629 INFO:teuthology.orchestra.run.smithi195.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-03-07T02:50:06.633 INFO:teuthology.orchestra.run.smithi195.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-03-07T02:50:06.633 INFO:teuthology.orchestra.run.smithi195.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-03-07T02:50:06.639 INFO:teuthology.orchestra.run.smithi148.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-03-07T02:50:06.659 INFO:teuthology.orchestra.run.smithi195.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-03-07T02:50:06.659 INFO:teuthology.orchestra.run.smithi195.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-03-07T02:50:06.659 INFO:teuthology.orchestra.run.smithi195.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-03-07T02:50:06.660 INFO:teuthology.orchestra.run.smithi195.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-03-07T02:50:06.661 INFO:teuthology.orchestra.run.smithi195.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-03-07T02:50:06.661 INFO:teuthology.orchestra.run.smithi148.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-03-07T02:50:06.668 INFO:teuthology.orchestra.run.smithi195.stdout:Get:64 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-03-07T02:50:06.668 INFO:teuthology.orchestra.run.smithi195.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-03-07T02:50:06.672 INFO:teuthology.orchestra.run.smithi148.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-03-07T02:50:06.687 INFO:teuthology.orchestra.run.smithi195.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-03-07T02:50:06.692 INFO:teuthology.orchestra.run.smithi195.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-03-07T02:50:06.694 INFO:teuthology.orchestra.run.smithi195.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-03-07T02:50:06.695 INFO:teuthology.orchestra.run.smithi195.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-03-07T02:50:06.696 INFO:teuthology.orchestra.run.smithi195.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-03-07T02:50:06.700 INFO:teuthology.orchestra.run.smithi195.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-03-07T02:50:06.703 INFO:teuthology.orchestra.run.smithi195.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-03-07T02:50:06.708 INFO:teuthology.orchestra.run.smithi195.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-03-07T02:50:06.716 INFO:teuthology.orchestra.run.smithi195.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-03-07T02:50:06.717 INFO:teuthology.orchestra.run.smithi195.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-03-07T02:50:06.736 INFO:teuthology.orchestra.run.smithi195.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-03-07T02:50:06.744 INFO:teuthology.orchestra.run.smithi195.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-03-07T02:50:06.744 INFO:teuthology.orchestra.run.smithi195.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-03-07T02:50:06.745 INFO:teuthology.orchestra.run.smithi195.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-03-07T02:50:06.771 INFO:teuthology.orchestra.run.smithi148.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-03-07T02:50:06.785 INFO:teuthology.orchestra.run.smithi148.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-03-07T02:50:06.795 INFO:teuthology.orchestra.run.smithi148.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-03-07T02:50:06.798 INFO:teuthology.orchestra.run.smithi148.stdout:Get:70 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-03-07T02:50:06.816 INFO:teuthology.orchestra.run.smithi148.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-03-07T02:50:06.825 INFO:teuthology.orchestra.run.smithi148.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-03-07T02:50:06.839 INFO:teuthology.orchestra.run.smithi148.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-03-07T02:50:06.847 INFO:teuthology.orchestra.run.smithi148.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-03-07T02:50:06.848 INFO:teuthology.orchestra.run.smithi148.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-03-07T02:50:06.852 INFO:teuthology.orchestra.run.smithi148.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-03-07T02:50:06.869 INFO:teuthology.orchestra.run.smithi148.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-03-07T02:50:06.870 INFO:teuthology.orchestra.run.smithi148.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-03-07T02:50:06.871 INFO:teuthology.orchestra.run.smithi148.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-03-07T02:50:07.575 INFO:teuthology.orchestra.run.smithi148.stdout:Get:80 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 18.2.1-707-gffcfe7db-1jammy [3,105 kB] 2024-03-07T02:50:07.688 INFO:teuthology.orchestra.run.smithi195.stdout:Get:80 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 18.2.1-707-gffcfe7db-1jammy [3,105 kB] 2024-03-07T02:50:08.265 INFO:teuthology.orchestra.run.smithi148.stdout:Get:81 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 18.2.1-707-gffcfe7db-1jammy [672 kB] 2024-03-07T02:50:08.388 INFO:teuthology.orchestra.run.smithi148.stdout:Get:82 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 18.2.1-707-gffcfe7db-1jammy [356 kB] 2024-03-07T02:50:08.452 INFO:teuthology.orchestra.run.smithi148.stdout:Get:83 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 18.2.1-707-gffcfe7db-1jammy [32.8 kB] 2024-03-07T02:50:08.479 INFO:teuthology.orchestra.run.smithi195.stdout:Get:81 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 18.2.1-707-gffcfe7db-1jammy [672 kB] 2024-03-07T02:50:08.489 INFO:teuthology.orchestra.run.smithi148.stdout:Get:84 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 18.2.1-707-gffcfe7db-1jammy [180 kB] 2024-03-07T02:50:08.504 INFO:teuthology.orchestra.run.smithi148.stdout:Get:85 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 18.2.1-707-gffcfe7db-1jammy [62.1 kB] 2024-03-07T02:50:08.509 INFO:teuthology.orchestra.run.smithi148.stdout:Get:86 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 18.2.1-707-gffcfe7db-1jammy [330 kB] 2024-03-07T02:50:08.608 INFO:teuthology.orchestra.run.smithi148.stdout:Get:87 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 18.2.1-707-gffcfe7db-1jammy [5,887 kB] 2024-03-07T02:50:08.653 INFO:teuthology.orchestra.run.smithi195.stdout:Get:82 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 18.2.1-707-gffcfe7db-1jammy [356 kB] 2024-03-07T02:50:08.721 INFO:teuthology.orchestra.run.smithi195.stdout:Get:83 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 18.2.1-707-gffcfe7db-1jammy [32.8 kB] 2024-03-07T02:50:08.724 INFO:teuthology.orchestra.run.smithi195.stdout:Get:84 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 18.2.1-707-gffcfe7db-1jammy [180 kB] 2024-03-07T02:50:08.779 INFO:teuthology.orchestra.run.smithi195.stdout:Get:85 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 18.2.1-707-gffcfe7db-1jammy [62.1 kB] 2024-03-07T02:50:08.785 INFO:teuthology.orchestra.run.smithi195.stdout:Get:86 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 18.2.1-707-gffcfe7db-1jammy [330 kB] 2024-03-07T02:50:08.896 INFO:teuthology.orchestra.run.smithi195.stdout:Get:87 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 18.2.1-707-gffcfe7db-1jammy [5,887 kB] 2024-03-07T02:50:09.807 INFO:teuthology.orchestra.run.smithi148.stdout:Get:88 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 18.2.1-707-gffcfe7db-1jammy [112 kB] 2024-03-07T02:50:09.859 INFO:teuthology.orchestra.run.smithi148.stdout:Get:89 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 18.2.1-707-gffcfe7db-1jammy [441 kB] 2024-03-07T02:50:09.943 INFO:teuthology.orchestra.run.smithi148.stdout:Get:90 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 18.2.1-707-gffcfe7db-1jammy [22.0 MB] 2024-03-07T02:50:10.486 INFO:teuthology.orchestra.run.smithi195.stdout:Get:88 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 18.2.1-707-gffcfe7db-1jammy [112 kB] 2024-03-07T02:50:10.551 INFO:teuthology.orchestra.run.smithi195.stdout:Get:89 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 18.2.1-707-gffcfe7db-1jammy [441 kB] 2024-03-07T02:50:10.693 INFO:teuthology.orchestra.run.smithi195.stdout:Get:90 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 18.2.1-707-gffcfe7db-1jammy [22.0 MB] 2024-03-07T02:50:14.531 INFO:teuthology.orchestra.run.smithi148.stdout:Get:91 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 18.2.1-707-gffcfe7db-1jammy [6,849 kB] 2024-03-07T02:50:15.599 INFO:teuthology.orchestra.run.smithi148.stdout:Get:92 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 18.2.1-707-gffcfe7db-1jammy [240 kB] 2024-03-07T02:50:15.662 INFO:teuthology.orchestra.run.smithi148.stdout:Get:93 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 18.2.1-707-gffcfe7db-1jammy [130 kB] 2024-03-07T02:50:15.671 INFO:teuthology.orchestra.run.smithi148.stdout:Get:94 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 18.2.1-707-gffcfe7db-1jammy [1,484 kB] 2024-03-07T02:50:15.968 INFO:teuthology.orchestra.run.smithi148.stdout:Get:95 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 18.2.1-707-gffcfe7db-1jammy [9,906 kB] 2024-03-07T02:50:17.948 INFO:teuthology.orchestra.run.smithi195.stdout:Get:91 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 18.2.1-707-gffcfe7db-1jammy [6,849 kB] 2024-03-07T02:50:18.293 INFO:teuthology.orchestra.run.smithi148.stdout:Get:96 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 18.2.1-707-gffcfe7db-1jammy [31.6 MB] 2024-03-07T02:50:19.953 INFO:teuthology.orchestra.run.smithi195.stdout:Get:92 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 18.2.1-707-gffcfe7db-1jammy [240 kB] 2024-03-07T02:50:20.030 INFO:teuthology.orchestra.run.smithi195.stdout:Get:93 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 18.2.1-707-gffcfe7db-1jammy [130 kB] 2024-03-07T02:50:20.082 INFO:teuthology.orchestra.run.smithi195.stdout:Get:94 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 18.2.1-707-gffcfe7db-1jammy [1,484 kB] 2024-03-07T02:50:20.581 INFO:teuthology.orchestra.run.smithi195.stdout:Get:95 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 18.2.1-707-gffcfe7db-1jammy [9,906 kB] 2024-03-07T02:50:24.084 INFO:teuthology.orchestra.run.smithi195.stdout:Get:96 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 18.2.1-707-gffcfe7db-1jammy [31.6 MB] 2024-03-07T02:50:25.087 INFO:teuthology.orchestra.run.smithi148.stdout:Get:97 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 18.2.1-707-gffcfe7db-1jammy [14.1 kB] 2024-03-07T02:50:25.087 INFO:teuthology.orchestra.run.smithi148.stdout:Get:98 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 18.2.1-707-gffcfe7db-1jammy [824 kB] 2024-03-07T02:50:25.161 INFO:teuthology.orchestra.run.smithi148.stdout:Get:99 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 18.2.1-707-gffcfe7db-1jammy [2,175 kB] 2024-03-07T02:50:25.623 INFO:teuthology.orchestra.run.smithi148.stdout:Get:100 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 18.2.1-707-gffcfe7db-1jammy [237 kB] 2024-03-07T02:50:25.668 INFO:teuthology.orchestra.run.smithi148.stdout:Get:101 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 18.2.1-707-gffcfe7db-1jammy [143 kB] 2024-03-07T02:50:25.699 INFO:teuthology.orchestra.run.smithi148.stdout:Get:102 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 18.2.1-707-gffcfe7db-1jammy [1,880 kB] 2024-03-07T02:50:26.050 INFO:teuthology.orchestra.run.smithi148.stdout:Get:103 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 18.2.1-707-gffcfe7db-1jammy [8,625 kB] 2024-03-07T02:50:27.596 INFO:teuthology.orchestra.run.smithi148.stdout:Get:104 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 18.2.1-707-gffcfe7db-1jammy [14.3 kB] 2024-03-07T02:50:27.597 INFO:teuthology.orchestra.run.smithi148.stdout:Get:105 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 18.2.1-707-gffcfe7db-1jammy [46.6 MB] 2024-03-07T02:50:35.726 INFO:teuthology.orchestra.run.smithi195.stdout:Get:97 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 18.2.1-707-gffcfe7db-1jammy [14.1 kB] 2024-03-07T02:50:35.726 INFO:teuthology.orchestra.run.smithi195.stdout:Get:98 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 18.2.1-707-gffcfe7db-1jammy [824 kB] 2024-03-07T02:50:35.851 INFO:teuthology.orchestra.run.smithi195.stdout:Get:99 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 18.2.1-707-gffcfe7db-1jammy [2,175 kB] 2024-03-07T02:50:36.367 INFO:teuthology.orchestra.run.smithi195.stdout:Get:100 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 18.2.1-707-gffcfe7db-1jammy [237 kB] 2024-03-07T02:50:36.437 INFO:teuthology.orchestra.run.smithi195.stdout:Get:101 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 18.2.1-707-gffcfe7db-1jammy [143 kB] 2024-03-07T02:50:36.466 INFO:teuthology.orchestra.run.smithi195.stdout:Get:102 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 18.2.1-707-gffcfe7db-1jammy [1,880 kB] 2024-03-07T02:50:36.916 INFO:teuthology.orchestra.run.smithi195.stdout:Get:103 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 18.2.1-707-gffcfe7db-1jammy [8,625 kB] 2024-03-07T02:50:38.405 INFO:teuthology.orchestra.run.smithi148.stdout:Get:106 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 18.2.1-707-gffcfe7db-1jammy [121 kB] 2024-03-07T02:50:38.407 INFO:teuthology.orchestra.run.smithi148.stdout:Get:107 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 18.2.1-707-gffcfe7db-1jammy [38.3 kB] 2024-03-07T02:50:38.471 INFO:teuthology.orchestra.run.smithi148.stdout:Get:108 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 18.2.1-707-gffcfe7db-1jammy [11.6 MB] 2024-03-07T02:50:38.825 INFO:teuthology.orchestra.run.smithi195.stdout:Get:104 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 18.2.1-707-gffcfe7db-1jammy [14.3 kB] 2024-03-07T02:50:38.825 INFO:teuthology.orchestra.run.smithi195.stdout:Get:105 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 18.2.1-707-gffcfe7db-1jammy [46.6 MB] 2024-03-07T02:50:41.767 INFO:teuthology.orchestra.run.smithi148.stdout:Get:109 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 18.2.1-707-gffcfe7db-1jammy [93.1 kB] 2024-03-07T02:50:42.326 INFO:teuthology.orchestra.run.smithi148.stdout:Fetched 181 MB in 36s (5,032 kB/s) 2024-03-07T02:50:42.471 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libfmt8:amd64. 2024-03-07T02:50:42.567 INFO:teuthology.orchestra.run.smithi148.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165994 files and directories currently installed.) 2024-03-07T02:50:42.573 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-03-07T02:50:42.662 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:50:42.997 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-03-07T02:50:43.024 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-03-07T02:50:43.064 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:50:43.426 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-03-07T02:50:43.452 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../002-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-03-07T02:50:43.492 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:50:43.812 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-03-07T02:50:43.839 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../003-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-03-07T02:50:43.879 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:50:44.324 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-03-07T02:50:44.351 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../004-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:50:44.399 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:44.793 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-03-07T02:50:44.820 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../005-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:50:44.860 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:45.205 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-03-07T02:50:45.232 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../006-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:50:45.271 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:45.574 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-03-07T02:50:45.601 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../007-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-03-07T02:50:45.632 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:50:45.969 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package librados2. 2024-03-07T02:50:45.996 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../008-librados2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:46.037 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:46.482 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package librbd1. 2024-03-07T02:50:46.508 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../009-librbd1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:46.548 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:46.935 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libcephfs2. 2024-03-07T02:50:46.961 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../010-libcephfs2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:47.001 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:47.278 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-rados. 2024-03-07T02:50:47.306 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../011-python3-rados_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:47.345 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:47.673 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-03-07T02:50:47.700 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../012-python3-ceph-argparse_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:47.739 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.008 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-cephfs. 2024-03-07T02:50:48.024 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../013-python3-cephfs_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:48.058 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.336 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-ceph-common. 2024-03-07T02:50:48.363 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../014-python3-ceph-common_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:50:48.402 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:48.731 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-wcwidth. 2024-03-07T02:50:48.758 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-03-07T02:50:48.797 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:50:49.117 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-prettytable. 2024-03-07T02:50:49.143 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-03-07T02:50:49.183 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-03-07T02:50:49.444 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-rbd. 2024-03-07T02:50:49.471 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../017-python3-rbd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:49.511 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:49.806 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-03-07T02:50:49.832 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-03-07T02:50:49.872 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:50:50.200 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package librgw2. 2024-03-07T02:50:50.227 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../019-librgw2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:50.274 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:50.777 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-rgw. 2024-03-07T02:50:50.804 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../020-python3-rgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:50.838 INFO:teuthology.orchestra.run.smithi195.stdout:Get:106 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 18.2.1-707-gffcfe7db-1jammy [121 kB] 2024-03-07T02:50:50.841 INFO:teuthology.orchestra.run.smithi195.stdout:Get:107 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 18.2.1-707-gffcfe7db-1jammy [38.3 kB] 2024-03-07T02:50:50.843 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:50.906 INFO:teuthology.orchestra.run.smithi195.stdout:Get:108 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 18.2.1-707-gffcfe7db-1jammy [11.6 MB] 2024-03-07T02:50:51.164 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package liboath0:amd64. 2024-03-07T02:50:51.190 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-03-07T02:50:51.230 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:50:51.508 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libradosstriper1. 2024-03-07T02:50:51.535 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../022-libradosstriper1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:51.574 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:51.977 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-common. 2024-03-07T02:50:52.004 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../023-ceph-common_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:52.044 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:53.283 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-base. 2024-03-07T02:50:53.309 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../024-ceph-base_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:53.359 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:54.087 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-03-07T02:50:54.114 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../025-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-03-07T02:50:54.153 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:50:54.408 INFO:teuthology.orchestra.run.smithi195.stdout:Get:109 https://3.chacra.ceph.com/r/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 18.2.1-707-gffcfe7db-1jammy [93.1 kB] 2024-03-07T02:50:54.507 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-cheroot. 2024-03-07T02:50:54.534 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../026-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-03-07T02:50:54.582 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:50:54.910 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-03-07T02:50:54.937 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../027-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-03-07T02:50:54.976 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:50:55.009 INFO:teuthology.orchestra.run.smithi195.stdout:Fetched 181 MB in 49s (3,724 kB/s) 2024-03-07T02:50:55.151 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libfmt8:amd64. 2024-03-07T02:50:55.237 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-jaraco.text. 2024-03-07T02:50:55.240 INFO:teuthology.orchestra.run.smithi195.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165994 files and directories currently installed.) 2024-03-07T02:50:55.246 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-03-07T02:50:55.264 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../028-python3-jaraco.text_3.6.0-2_all.deb ... 2024-03-07T02:50:55.304 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:50:55.317 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:50:55.615 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-03-07T02:50:55.642 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../029-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-03-07T02:50:55.653 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-03-07T02:50:55.680 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-03-07T02:50:55.682 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:50:55.720 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:50:55.985 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-tempora. 2024-03-07T02:50:56.012 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../030-python3-tempora_4.1.2-1_all.deb ... 2024-03-07T02:50:56.051 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-03-07T02:50:56.081 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-03-07T02:50:56.108 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../002-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-03-07T02:50:56.148 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:50:56.329 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-portend. 2024-03-07T02:50:56.356 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../031-python3-portend_3.0.0-1_all.deb ... 2024-03-07T02:50:56.387 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-03-07T02:50:56.476 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-03-07T02:50:56.502 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../003-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-03-07T02:50:56.542 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:50:56.682 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-03-07T02:50:56.709 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../032-python3-zc.lockfile_2.0-1_all.deb ... 2024-03-07T02:50:56.748 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-03-07T02:50:56.971 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-03-07T02:50:56.997 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../004-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:50:57.046 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:57.051 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-cherrypy3. 2024-03-07T02:50:57.078 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../033-python3-cherrypy3_18.6.1-4_all.deb ... 2024-03-07T02:50:57.118 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:50:57.441 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-03-07T02:50:57.454 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-natsort. 2024-03-07T02:50:57.467 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../005-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:50:57.481 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../034-python3-natsort_8.0.2-1_all.deb ... 2024-03-07T02:50:57.507 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:57.529 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-03-07T02:50:57.835 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-03-07T02:50:57.841 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-logutils. 2024-03-07T02:50:57.861 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../006-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-03-07T02:50:57.867 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../035-python3-logutils_0.3.3-8_all.deb ... 2024-03-07T02:50:57.893 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:50:57.970 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-03-07T02:50:58.213 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-03-07T02:50:58.239 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../007-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-03-07T02:50:58.280 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:50:58.302 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-markupsafe. 2024-03-07T02:50:58.328 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../036-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-03-07T02:50:58.368 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:50:58.608 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package librados2. 2024-03-07T02:50:58.634 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../008-librados2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:58.646 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-mako. 2024-03-07T02:50:58.673 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../037-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-03-07T02:50:58.674 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:58.721 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:50:58.991 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-simplegeneric. 2024-03-07T02:50:59.017 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../038-python3-simplegeneric_0.8.1-3_all.deb ... 2024-03-07T02:50:59.049 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:50:59.127 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package librbd1. 2024-03-07T02:50:59.153 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../009-librbd1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:59.194 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:50:59.352 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-singledispatch. 2024-03-07T02:50:59.378 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../039-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-03-07T02:50:59.418 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:50:59.713 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libcephfs2. 2024-03-07T02:50:59.721 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-webob. 2024-03-07T02:50:59.739 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../010-libcephfs2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:50:59.747 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../040-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-03-07T02:50:59.787 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:50:59.863 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:00.141 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-waitress. 2024-03-07T02:51:00.167 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../041-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-03-07T02:51:00.174 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-rados. 2024-03-07T02:51:00.200 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../011-python3-rados_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:00.211 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:51:00.232 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:00.502 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-tempita. 2024-03-07T02:51:00.517 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../042-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-03-07T02:51:00.552 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:51:00.560 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-03-07T02:51:00.586 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../012-python3-ceph-argparse_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:00.626 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:00.879 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-cephfs. 2024-03-07T02:51:00.880 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-paste. 2024-03-07T02:51:00.894 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../013-python3-cephfs_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:00.906 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../043-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-03-07T02:51:00.937 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:00.954 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:51:01.257 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-ceph-common. 2024-03-07T02:51:01.274 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-03-07T02:51:01.283 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../014-python3-ceph-common_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:01.290 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../044-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-03-07T02:51:01.323 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:01.332 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:51:01.635 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-wcwidth. 2024-03-07T02:51:01.650 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-03-07T02:51:01.669 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pastedeploy. 2024-03-07T02:51:01.685 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:51:01.695 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../045-python3-pastedeploy_2.1.1-1_all.deb ... 2024-03-07T02:51:01.768 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:51:01.937 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-prettytable. 2024-03-07T02:51:01.952 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-03-07T02:51:01.987 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-03-07T02:51:02.105 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-webtest. 2024-03-07T02:51:02.132 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../046-python3-webtest_2.0.35-1_all.deb ... 2024-03-07T02:51:02.180 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-03-07T02:51:02.248 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-rbd. 2024-03-07T02:51:02.263 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../017-python3-rbd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:02.306 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:02.508 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pecan. 2024-03-07T02:51:02.535 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../047-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-03-07T02:51:02.574 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:51:02.676 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-03-07T02:51:02.703 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-03-07T02:51:02.743 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:51:02.904 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-certifi. 2024-03-07T02:51:02.931 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../048-python3-certifi_2020.6.20-1_all.deb ... 2024-03-07T02:51:02.978 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-03-07T02:51:03.054 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package librgw2. 2024-03-07T02:51:03.080 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../019-librgw2_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:03.112 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:03.299 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-idna. 2024-03-07T02:51:03.325 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../049-python3-idna_3.3-1_all.deb ... 2024-03-07T02:51:03.365 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-idna (3.3-1) ... 2024-03-07T02:51:03.573 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-rgw. 2024-03-07T02:51:03.588 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../020-python3-rgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:03.631 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:03.676 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-urllib3. 2024-03-07T02:51:03.703 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../050-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-03-07T02:51:03.742 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:51:04.001 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package liboath0:amd64. 2024-03-07T02:51:04.027 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-03-07T02:51:04.062 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-requests. 2024-03-07T02:51:04.067 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:51:04.080 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../051-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-03-07T02:51:04.120 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:51:04.404 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libradosstriper1. 2024-03-07T02:51:04.430 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../022-libradosstriper1_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:04.432 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-werkzeug. 2024-03-07T02:51:04.459 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../052-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ... 2024-03-07T02:51:04.470 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:04.499 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:51:04.776 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-03-07T02:51:04.792 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../053-ceph-mgr-modules-core_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:04.815 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-common. 2024-03-07T02:51:04.834 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:04.841 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../023-ceph-common_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:04.873 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:05.181 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-03-07T02:51:05.196 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../054-libsqlite3-mod-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:05.239 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:05.558 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-openssl. 2024-03-07T02:51:05.585 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../055-python3-openssl_21.0.0-1_all.deb ... 2024-03-07T02:51:05.625 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-03-07T02:51:05.961 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mgr. 2024-03-07T02:51:05.988 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../056-ceph-mgr_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:06.028 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:06.171 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-base. 2024-03-07T02:51:06.197 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../024-ceph-base_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:06.247 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:06.406 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mon. 2024-03-07T02:51:06.433 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../057-ceph-mon_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:06.464 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:06.943 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-03-07T02:51:06.968 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../025-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-03-07T02:51:07.008 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:51:07.345 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-cheroot. 2024-03-07T02:51:07.371 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../026-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-03-07T02:51:07.392 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-osd. 2024-03-07T02:51:07.419 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:51:07.420 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../058-ceph-osd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:07.459 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:07.798 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-03-07T02:51:07.824 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../027-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-03-07T02:51:07.872 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:51:08.234 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-jaraco.text. 2024-03-07T02:51:08.260 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../028-python3-jaraco.text_3.6.0-2_all.deb ... 2024-03-07T02:51:08.300 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:51:08.603 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-03-07T02:51:08.629 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../029-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-03-07T02:51:08.670 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:51:09.006 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-tempora. 2024-03-07T02:51:09.032 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../030-python3-tempora_4.1.2-1_all.deb ... 2024-03-07T02:51:09.081 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-03-07T02:51:09.254 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph. 2024-03-07T02:51:09.281 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../059-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:09.320 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:09.451 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-portend. 2024-03-07T02:51:09.477 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../031-python3-portend_3.0.0-1_all.deb ... 2024-03-07T02:51:09.525 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-03-07T02:51:09.665 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-fuse. 2024-03-07T02:51:09.692 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../060-ceph-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:09.740 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:10.028 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-03-07T02:51:10.054 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../032-python3-zc.lockfile_2.0-1_all.deb ... 2024-03-07T02:51:10.095 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-03-07T02:51:10.101 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mds. 2024-03-07T02:51:10.128 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../061-ceph-mds_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:10.167 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:10.398 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-cherrypy3. 2024-03-07T02:51:10.424 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../033-python3-cherrypy3_18.6.1-4_all.deb ... 2024-03-07T02:51:10.464 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:51:10.529 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package cephadm. 2024-03-07T02:51:10.555 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../062-cephadm_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:10.587 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:10.802 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-natsort. 2024-03-07T02:51:10.828 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../034-python3-natsort_8.0.2-1_all.deb ... 2024-03-07T02:51:10.859 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-03-07T02:51:10.899 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-asyncssh. 2024-03-07T02:51:10.925 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../063-python3-asyncssh_2.5.0-1_all.deb ... 2024-03-07T02:51:10.998 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-03-07T02:51:11.163 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-logutils. 2024-03-07T02:51:11.189 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../035-python3-logutils_0.3.3-8_all.deb ... 2024-03-07T02:51:11.231 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-03-07T02:51:11.293 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python-babel-localedata. 2024-03-07T02:51:11.308 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:51:11.351 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:51:11.515 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-markupsafe. 2024-03-07T02:51:11.542 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../036-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-03-07T02:51:11.582 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:51:11.918 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-mako. 2024-03-07T02:51:11.944 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../037-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-03-07T02:51:11.985 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:51:12.296 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-simplegeneric. 2024-03-07T02:51:12.323 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../038-python3-simplegeneric_0.8.1-3_all.deb ... 2024-03-07T02:51:12.354 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:51:12.364 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-babel. 2024-03-07T02:51:12.391 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:51:12.431 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:51:12.659 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-singledispatch. 2024-03-07T02:51:12.683 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../039-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-03-07T02:51:12.723 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:51:12.776 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-jinja2. 2024-03-07T02:51:12.802 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ... 2024-03-07T02:51:12.842 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:51:13.010 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-webob. 2024-03-07T02:51:13.036 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../040-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-03-07T02:51:13.090 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:51:13.204 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-03-07T02:51:13.230 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:13.304 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:13.404 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-waitress. 2024-03-07T02:51:13.431 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../041-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-03-07T02:51:13.466 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:51:13.641 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-bcrypt. 2024-03-07T02:51:13.667 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../068-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-03-07T02:51:13.707 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:51:13.774 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-tempita. 2024-03-07T02:51:13.800 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../042-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-03-07T02:51:13.840 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:51:14.010 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pluggy. 2024-03-07T02:51:14.036 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../069-python3-pluggy_0.13.0-7.1_all.deb ... 2024-03-07T02:51:14.076 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:51:14.152 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-paste. 2024-03-07T02:51:14.178 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../043-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-03-07T02:51:14.218 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:51:14.379 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-py. 2024-03-07T02:51:14.406 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../070-python3-py_1.10.0-1_all.deb ... 2024-03-07T02:51:14.446 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-py (1.10.0-1) ... 2024-03-07T02:51:14.563 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-03-07T02:51:14.589 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../044-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-03-07T02:51:14.629 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:51:14.741 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-iniconfig. 2024-03-07T02:51:14.768 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../071-python3-iniconfig_1.1.1-2_all.deb ... 2024-03-07T02:51:14.799 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-03-07T02:51:14.932 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pastedeploy. 2024-03-07T02:51:14.959 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../045-python3-pastedeploy_2.1.1-1_all.deb ... 2024-03-07T02:51:14.999 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:51:15.069 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-toml. 2024-03-07T02:51:15.095 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../072-python3-toml_0.10.2-1_all.deb ... 2024-03-07T02:51:15.135 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-03-07T02:51:15.419 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-webtest. 2024-03-07T02:51:15.438 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pytest. 2024-03-07T02:51:15.445 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../046-python3-webtest_2.0.35-1_all.deb ... 2024-03-07T02:51:15.465 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../073-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-03-07T02:51:15.477 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-03-07T02:51:15.504 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:51:15.763 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pecan. 2024-03-07T02:51:15.790 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../047-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-03-07T02:51:15.830 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:51:15.908 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-repoze.lru. 2024-03-07T02:51:15.935 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../074-python3-repoze.lru_0.7-2_all.deb ... 2024-03-07T02:51:15.975 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-03-07T02:51:16.201 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-certifi. 2024-03-07T02:51:16.229 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../048-python3-certifi_2020.6.20-1_all.deb ... 2024-03-07T02:51:16.268 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-03-07T02:51:16.278 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-routes. 2024-03-07T02:51:16.305 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../075-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-03-07T02:51:16.344 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:51:16.596 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-idna. 2024-03-07T02:51:16.622 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../049-python3-idna_3.3-1_all.deb ... 2024-03-07T02:51:16.662 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-idna (3.3-1) ... 2024-03-07T02:51:16.689 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-03-07T02:51:16.716 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../076-ceph-mgr-dashboard_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:16.756 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:16.982 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-urllib3. 2024-03-07T02:51:17.008 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../050-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-03-07T02:51:17.048 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:51:17.126 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-03-07T02:51:17.141 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../077-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-03-07T02:51:17.175 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:51:17.360 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-requests. 2024-03-07T02:51:17.387 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../051-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-03-07T02:51:17.426 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:51:17.688 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-joblib. 2024-03-07T02:51:17.704 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-werkzeug. 2024-03-07T02:51:17.714 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../078-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-03-07T02:51:17.731 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../052-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ... 2024-03-07T02:51:17.754 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:51:17.762 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:51:18.057 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-03-07T02:51:18.084 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../053-ceph-mgr-modules-core_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:18.091 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-03-07T02:51:18.118 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../079-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-03-07T02:51:18.123 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:18.157 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:51:18.419 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-sklearn. 2024-03-07T02:51:18.446 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../080-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-03-07T02:51:18.485 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:51:18.522 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-03-07T02:51:18.549 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../054-libsqlite3-mod-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:18.589 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:18.866 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-openssl. 2024-03-07T02:51:18.893 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../055-python3-openssl_21.0.0-1_all.deb ... 2024-03-07T02:51:18.933 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-03-07T02:51:19.169 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-03-07T02:51:19.197 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../081-ceph-mgr-diskprediction-local_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:19.235 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:19.261 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mgr. 2024-03-07T02:51:19.287 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../056-ceph-mgr_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:19.336 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:19.764 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mon. 2024-03-07T02:51:19.791 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../057-ceph-mon_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:19.830 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:20.197 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-cachetools. 2024-03-07T02:51:20.224 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../082-python3-cachetools_5.0.0-1_all.deb ... 2024-03-07T02:51:20.263 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-03-07T02:51:20.684 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pyasn1. 2024-03-07T02:51:20.712 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../083-python3-pyasn1_0.4.8-1_all.deb ... 2024-03-07T02:51:20.751 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:51:20.767 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-osd. 2024-03-07T02:51:20.794 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../058-ceph-osd_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:20.834 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:21.079 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-03-07T02:51:21.107 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../084-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-03-07T02:51:21.145 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:51:21.449 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-rsa. 2024-03-07T02:51:21.476 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../085-python3-rsa_4.8-1_all.deb ... 2024-03-07T02:51:21.532 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-rsa (4.8-1) ... 2024-03-07T02:51:21.852 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-google-auth. 2024-03-07T02:51:21.880 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../086-python3-google-auth_1.5.1-3_all.deb ... 2024-03-07T02:51:21.919 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-03-07T02:51:22.280 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-03-07T02:51:22.308 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../087-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-03-07T02:51:22.355 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:51:22.679 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph. 2024-03-07T02:51:22.683 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-websocket. 2024-03-07T02:51:22.705 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../059-ceph_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:22.711 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../088-python3-websocket_1.2.3-1_all.deb ... 2024-03-07T02:51:22.745 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:22.749 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-03-07T02:51:23.065 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-fuse. 2024-03-07T02:51:23.078 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-kubernetes. 2024-03-07T02:51:23.091 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../060-ceph-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:23.106 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../089-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-03-07T02:51:23.131 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:23.144 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:51:23.535 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mds. 2024-03-07T02:51:23.561 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../061-ceph-mds_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:23.601 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:23.793 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-03-07T02:51:23.810 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../090-ceph-mgr-k8sevents_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:23.851 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:23.946 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package cephadm. 2024-03-07T02:51:23.972 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../062-cephadm_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:24.004 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:24.179 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libonig5:amd64. 2024-03-07T02:51:24.207 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../091-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-03-07T02:51:24.246 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:51:24.323 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-asyncssh. 2024-03-07T02:51:24.350 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../063-python3-asyncssh_2.5.0-1_all.deb ... 2024-03-07T02:51:24.390 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-03-07T02:51:24.565 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libjq1:amd64. 2024-03-07T02:51:24.593 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../092-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:51:24.632 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:51:24.693 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python-babel-localedata. 2024-03-07T02:51:24.716 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:51:24.759 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:51:24.926 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package jq. 2024-03-07T02:51:24.942 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../093-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:51:24.984 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-03-07T02:51:25.262 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package socat. 2024-03-07T02:51:25.289 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../094-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-03-07T02:51:25.328 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:51:25.640 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package xmlstarlet. 2024-03-07T02:51:25.656 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../095-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-03-07T02:51:25.698 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:51:25.956 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-babel. 2024-03-07T02:51:25.984 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-03-07T02:51:26.022 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:51:26.052 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-test. 2024-03-07T02:51:26.068 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../096-ceph-test_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:26.111 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:26.351 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-jinja2. 2024-03-07T02:51:26.378 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ... 2024-03-07T02:51:26.417 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:51:26.779 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-03-07T02:51:26.806 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:26.845 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:27.199 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-bcrypt. 2024-03-07T02:51:27.226 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../068-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-03-07T02:51:27.265 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:51:27.568 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pluggy. 2024-03-07T02:51:27.595 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../069-python3-pluggy_0.13.0-7.1_all.deb ... 2024-03-07T02:51:27.634 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:51:27.905 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-py. 2024-03-07T02:51:27.932 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../070-python3-py_1.10.0-1_all.deb ... 2024-03-07T02:51:27.963 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-py (1.10.0-1) ... 2024-03-07T02:51:28.216 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-iniconfig. 2024-03-07T02:51:28.231 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../071-python3-iniconfig_1.1.1-2_all.deb ... 2024-03-07T02:51:28.266 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-03-07T02:51:28.323 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package ceph-volume. 2024-03-07T02:51:28.351 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../097-ceph-volume_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:28.389 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:28.569 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-toml. 2024-03-07T02:51:28.595 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../072-python3-toml_0.10.2-1_all.deb ... 2024-03-07T02:51:28.635 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-03-07T02:51:28.938 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pytest. 2024-03-07T02:51:28.952 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package libcephfs-dev. 2024-03-07T02:51:28.965 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../073-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-03-07T02:51:28.980 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../098-libcephfs-dev_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:29.005 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:51:29.018 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:29.347 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package nvme-cli. 2024-03-07T02:51:29.350 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-repoze.lru. 2024-03-07T02:51:29.374 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../099-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-03-07T02:51:29.377 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../074-python3-repoze.lru_0.7-2_all.deb ... 2024-03-07T02:51:29.416 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-03-07T02:51:29.438 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:51:29.735 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-routes. 2024-03-07T02:51:29.761 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../075-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-03-07T02:51:29.793 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-03-07T02:51:29.802 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:51:29.821 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../100-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-03-07T02:51:29.859 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:51:30.156 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-03-07T02:51:30.183 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../076-ceph-mgr-dashboard_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:30.222 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:30.246 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pastescript. 2024-03-07T02:51:30.273 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../101-python3-pastescript_2.0.2-4_all.deb ... 2024-03-07T02:51:30.312 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-03-07T02:51:30.592 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-03-07T02:51:30.608 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../077-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-03-07T02:51:30.632 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-psutil. 2024-03-07T02:51:30.650 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:51:30.660 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../102-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-03-07T02:51:30.699 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-03-07T02:51:31.035 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pygments. 2024-03-07T02:51:31.063 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../103-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-03-07T02:51:31.102 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:51:31.330 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-joblib. 2024-03-07T02:51:31.356 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../078-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-03-07T02:51:31.396 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:51:31.556 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-pyinotify. 2024-03-07T02:51:31.583 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-03-07T02:51:31.622 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:51:31.716 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-03-07T02:51:31.732 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../079-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-03-07T02:51:31.766 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:51:31.933 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package python3-simplejson. 2024-03-07T02:51:31.961 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../105-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-03-07T02:51:31.999 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:51:32.044 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-sklearn. 2024-03-07T02:51:32.071 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../080-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-03-07T02:51:32.110 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:51:32.277 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-03-07T02:51:32.293 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../106-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-03-07T02:51:32.327 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:51:32.844 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-03-07T02:51:32.871 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../081-ceph-mgr-diskprediction-local_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:32.918 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:33.023 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package radosgw. 2024-03-07T02:51:33.050 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../107-radosgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:33.089 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:33.809 INFO:teuthology.orchestra.run.smithi148.stdout:Selecting previously unselected package rbd-fuse. 2024-03-07T02:51:33.825 INFO:teuthology.orchestra.run.smithi148.stdout:Preparing to unpack .../108-rbd-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:33.867 INFO:teuthology.orchestra.run.smithi148.stdout:Unpacking rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:33.939 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-cachetools. 2024-03-07T02:51:33.966 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../082-python3-cachetools_5.0.0-1_all.deb ... 2024-03-07T02:51:34.013 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-03-07T02:51:34.196 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-03-07T02:51:34.401 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pyasn1. 2024-03-07T02:51:34.428 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../083-python3-pyasn1_0.4.8-1_all.deb ... 2024-03-07T02:51:34.447 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:51:34.467 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:51:34.565 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:51:34.830 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-03-07T02:51:34.860 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-03-07T02:51:34.887 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../084-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-03-07T02:51:34.938 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:51:35.157 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-03-07T02:51:35.308 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-rsa. 2024-03-07T02:51:35.334 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../085-python3-rsa_4.8-1_all.deb ... 2024-03-07T02:51:35.382 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-rsa (4.8-1) ... 2024-03-07T02:51:35.596 INFO:teuthology.orchestra.run.smithi148.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-03-07T02:51:35.658 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:35.761 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-google-auth. 2024-03-07T02:51:35.788 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../086-python3-google-auth_1.5.1-3_all.deb ... 2024-03-07T02:51:35.836 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-03-07T02:51:36.156 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-03-07T02:51:36.166 INFO:teuthology.orchestra.run.smithi148.stdout:Adding system user cephadm....done 2024-03-07T02:51:36.183 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../087-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-03-07T02:51:36.222 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:51:36.314 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:51:36.534 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-websocket. 2024-03-07T02:51:36.561 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../088-python3-websocket_1.2.3-1_all.deb ... 2024-03-07T02:51:36.573 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:51:36.600 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-03-07T02:51:36.825 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:51:36.928 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-kubernetes. 2024-03-07T02:51:36.951 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:51:36.956 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../089-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-03-07T02:51:36.995 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:51:37.202 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-03-07T02:51:37.461 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:51:37.579 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-py (1.10.0-1) ... 2024-03-07T02:51:37.667 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-03-07T02:51:37.695 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../090-ceph-mgr-k8sevents_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:37.733 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:37.856 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:51:38.053 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libonig5:amd64. 2024-03-07T02:51:38.081 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../091-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-03-07T02:51:38.119 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:51:38.191 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-03-07T02:51:38.431 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libjq1:amd64. 2024-03-07T02:51:38.451 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-03-07T02:51:38.458 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../092-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:51:38.497 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:51:38.727 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:51:38.767 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package jq. 2024-03-07T02:51:38.794 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../093-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-03-07T02:51:38.833 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-03-07T02:51:38.978 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:51:39.094 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package socat. 2024-03-07T02:51:39.121 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../094-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-03-07T02:51:39.161 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:51:39.362 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:51:39.481 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package xmlstarlet. 2024-03-07T02:51:39.497 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../095-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-03-07T02:51:39.539 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:51:39.605 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:39.839 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:51:39.885 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-test. 2024-03-07T02:51:39.912 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../096-ceph-test_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:39.951 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:40.040 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:51:40.291 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:51:40.426 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-03-07T02:51:40.785 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:51:40.911 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-03-07T02:51:41.170 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:51:41.405 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:51:41.681 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:51:42.331 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package ceph-volume. 2024-03-07T02:51:42.358 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../097-ceph-volume_18.2.1-707-gffcfe7db-1jammy_all.deb ... 2024-03-07T02:51:42.375 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:51:42.397 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:42.626 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:51:42.726 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package libcephfs-dev. 2024-03-07T02:51:42.752 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:51:42.754 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../098-libcephfs-dev_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:42.792 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:42.894 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:51:43.171 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package nvme-cli. 2024-03-07T02:51:43.187 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-03-07T02:51:43.198 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../099-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-03-07T02:51:43.237 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:51:43.550 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-03-07T02:51:43.563 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-03-07T02:51:43.566 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../100-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-03-07T02:51:43.600 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:51:43.781 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:51:43.995 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pastescript. 2024-03-07T02:51:44.022 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../101-python3-pastescript_2.0.2-4_all.deb ... 2024-03-07T02:51:44.053 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-03-07T02:51:44.158 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:51:44.357 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-psutil. 2024-03-07T02:51:44.384 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../102-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-03-07T02:51:44.423 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-03-07T02:51:44.442 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:51:44.759 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pygments. 2024-03-07T02:51:44.777 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:51:44.787 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../103-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-03-07T02:51:44.826 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:51:45.028 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:51:45.146 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:45.255 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-pyinotify. 2024-03-07T02:51:45.283 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-03-07T02:51:45.313 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:51:45.405 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-idna (3.3-1) ... 2024-03-07T02:51:45.616 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package python3-simplejson. 2024-03-07T02:51:45.643 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../105-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-03-07T02:51:45.682 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:51:45.723 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:51:45.935 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-03-07T02:51:45.951 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../106-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-03-07T02:51:45.992 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:51:46.755 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package radosgw. 2024-03-07T02:51:46.783 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../107-radosgw_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:46.822 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:46.862 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:51:46.988 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-toml (0.10.2-1) ... 2024-03-07T02:51:47.222 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:51:47.331 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:51:47.458 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:51:47.558 INFO:teuthology.orchestra.run.smithi195.stdout:Selecting previously unselected package rbd-fuse. 2024-03-07T02:51:47.574 INFO:teuthology.orchestra.run.smithi195.stdout:Preparing to unpack .../108-rbd-fuse_18.2.1-707-gffcfe7db-1jammy_amd64.deb ... 2024-03-07T02:51:47.616 INFO:teuthology.orchestra.run.smithi195.stdout:Unpacking rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:47.767 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:51:47.919 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-03-07T02:51:48.052 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-03-07T02:51:48.170 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-03-07T02:51:48.272 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-03-07T02:51:48.303 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:51:48.429 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:51:48.522 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-03-07T02:51:48.530 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:51:48.806 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:51:48.872 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-03-07T02:51:49.057 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-03-07T02:51:49.307 INFO:teuthology.orchestra.run.smithi195.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-03-07T02:51:49.317 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-03-07T02:51:49.373 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:49.568 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:51:49.819 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-03-07T02:51:49.889 INFO:teuthology.orchestra.run.smithi195.stdout:Adding system user cephadm....done 2024-03-07T02:51:50.004 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-03-07T02:51:50.078 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:51:50.196 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-03-07T02:51:50.264 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-03-07T02:51:50.447 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:51:50.523 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-03-07T02:51:50.548 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:51:50.641 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-03-07T02:51:50.832 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:51:50.884 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-03-07T02:51:51.135 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-03-07T02:51:51.142 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:51:51.261 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-py (1.10.0-1) ... 2024-03-07T02:51:51.401 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:51:51.554 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-03-07T02:51:51.808 INFO:teuthology.orchestra.run.smithi148.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-03-07T02:51:51.881 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-03-07T02:51:52.062 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:51:52.116 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-03-07T02:51:52.163 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:51:52.384 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-03-07T02:51:52.540 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:51:52.634 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-03-07T02:51:52.799 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-portend (3.0.0-1) ... 2024-03-07T02:51:52.885 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-03-07T02:51:53.067 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:51:53.128 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-ceph-argparse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:53.193 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:51:53.379 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-03-07T02:51:53.505 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-03-07T02:51:53.519 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:51:53.748 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-03-07T02:51:53.820 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:51:53.874 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-03-07T02:51:54.088 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-03-07T02:51:54.189 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-rsa (4.8-1) ... 2024-03-07T02:51:54.242 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-03-07T02:51:54.368 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-03-07T02:51:54.457 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-03-07T02:51:54.635 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-03-07T02:51:54.733 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:51:54.895 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-03-07T02:51:54.992 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:51:55.196 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-03-07T02:51:55.394 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-03-07T02:51:55.687 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:51:55.831 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-03-07T02:51:56.014 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:51:56.074 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-03-07T02:51:56.123 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:56.200 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-03-07T02:51:56.249 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:56.326 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-03-07T02:51:56.375 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:56.501 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:56.627 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-03-07T02:51:56.627 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:56.847 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-03-07T02:51:57.020 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-03-07T02:51:57.132 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:57.258 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:57.262 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-03-07T02:51:57.384 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:57.597 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:51:57.597 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:51:57.656 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-03-07T02:51:57.940 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-03-07T02:51:58.069 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:58.195 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:58.275 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-03-07T02:51:58.304 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:51:58.526 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-03-07T02:51:58.652 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:58.937 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-idna (3.3-1) ... 2024-03-07T02:51:59.238 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-03-07T02:51:59.500 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:59.626 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:59.752 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:59.878 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:51:59.987 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:00.443 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:52:00.489 INFO:teuthology.orchestra.run.smithi148.stdout:Adding group ceph....done 2024-03-07T02:52:00.561 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-toml (0.10.2-1) ... 2024-03-07T02:52:00.812 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-03-07T02:52:00.874 INFO:teuthology.orchestra.run.smithi148.stdout:Adding system user ceph....done 2024-03-07T02:52:00.938 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-03-07T02:52:01.008 INFO:teuthology.orchestra.run.smithi148.stdout:Setting system user ceph properties....done 2024-03-07T02:52:01.025 INFO:teuthology.orchestra.run.smithi148.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-03-07T02:52:01.064 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-03-07T02:52:01.164 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-03-07T02:52:01.349 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-03-07T02:52:01.524 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-03-07T02:52:01.591 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-03-07T02:52:01.834 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:52:01.960 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-03-07T02:52:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:02.081 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:02.086 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-03-07T02:52:02.388 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-03-07T02:52:02.559 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:52:02.559 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:52:02.639 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-03-07T02:52:02.881 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-03-07T02:52:03.033 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:03.124 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-03-07T02:52:03.367 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-03-07T02:52:03.379 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-03-07T02:52:03.643 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-03-07T02:52:03.761 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-03-07T02:52:03.851 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:04.022 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-03-07T02:52:04.074 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:52:04.074 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:52:04.131 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-03-07T02:52:04.399 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-03-07T02:52:04.545 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:04.692 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-03-07T02:52:04.780 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:52:04.780 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:52:04.943 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-03-07T02:52:05.229 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:05.354 INFO:teuthology.orchestra.run.smithi195.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-03-07T02:52:05.487 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:52:05.488 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:52:05.612 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-03-07T02:52:05.722 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-03-07T02:52:05.973 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:06.090 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-03-07T02:52:06.099 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:06.267 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:06.333 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-portend (3.0.0-1) ... 2024-03-07T02:52:06.482 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:52:06.482 INFO:teuthology.orchestra.run.smithi148.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:52:06.584 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-03-07T02:52:06.709 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-03-07T02:52:06.968 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:07.044 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-03-07T02:52:07.111 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:07.236 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:07.345 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-03-07T02:52:07.379 INFO:teuthology.orchestra.run.smithi148.stdout:Setting up ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:07.626 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-03-07T02:52:07.715 INFO:teuthology.orchestra.run.smithi148.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-03-07T02:52:07.747 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-rsa (4.8-1) ... 2024-03-07T02:52:08.015 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-03-07T02:52:08.283 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-03-07T02:52:08.551 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-03-07T02:52:08.833 INFO:teuthology.orchestra.run.smithi148.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-03-07T02:52:08.944 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-03-07T02:52:09.220 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-03-07T02:52:09.306 INFO:teuthology.orchestra.run.smithi148.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-03-07T02:52:09.312 DEBUG:teuthology.parallel:result is None 2024-03-07T02:52:09.555 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-03-07T02:52:09.681 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up librados2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:09.807 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up librgw2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:09.933 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libsqlite3-mod-ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:10.059 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libcephfs2 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:10.185 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libradosstriper1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:10.388 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-03-07T02:52:10.682 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up librbd1 (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:10.808 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mgr-modules-core (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:10.934 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:11.144 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:52:11.144 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-03-07T02:52:11.652 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up libcephfs-dev (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:11.753 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-rados (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:11.853 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-03-07T02:52:13.050 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-rbd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:13.176 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up rbd-fuse (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:13.319 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-rgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:13.420 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up python3-cephfs (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:13.520 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-common (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:14.047 INFO:teuthology.orchestra.run.smithi195.stdout:Adding group ceph....done 2024-03-07T02:52:14.440 INFO:teuthology.orchestra.run.smithi195.stdout:Adding system user ceph....done 2024-03-07T02:52:14.600 INFO:teuthology.orchestra.run.smithi195.stdout:Setting system user ceph properties....done 2024-03-07T02:52:14.613 INFO:teuthology.orchestra.run.smithi195.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-03-07T02:52:14.754 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-03-07T02:52:15.065 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-03-07T02:52:15.530 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-test (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:15.656 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up radosgw (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:16.138 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:52:16.138 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-03-07T02:52:16.608 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-base (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:16.967 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-03-07T02:52:17.468 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mds (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:17.697 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:52:17.698 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-03-07T02:52:18.162 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mgr (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:18.379 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:52:18.379 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-03-07T02:52:18.846 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-osd (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:19.134 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:52:19.134 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-03-07T02:52:19.615 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mgr-k8sevents (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:19.757 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mgr-diskprediction-local (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:19.917 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mon (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:20.148 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:52:20.148 INFO:teuthology.orchestra.run.smithi195.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-03-07T02:52:20.693 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mgr-cephadm (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:20.828 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:20.995 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-mgr-dashboard (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:21.146 INFO:teuthology.orchestra.run.smithi195.stdout:Setting up ceph-volume (18.2.1-707-gffcfe7db-1jammy) ... 2024-03-07T02:52:21.516 INFO:teuthology.orchestra.run.smithi195.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-03-07T02:52:22.650 INFO:teuthology.orchestra.run.smithi195.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-03-07T02:52:23.128 INFO:teuthology.orchestra.run.smithi195.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-03-07T02:52:23.132 DEBUG:teuthology.parallel:result is None 2024-03-07T02:52:23.132 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:52:23.133 INFO:teuthology.packaging:ref: None 2024-03-07T02:52:23.133 INFO:teuthology.packaging:tag: None 2024-03-07T02:52:23.133 INFO:teuthology.packaging:branch: reef 2024-03-07T02:52:23.133 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:52:23.133 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:52:23.382 DEBUG:teuthology.orchestra.run.smithi148:> dpkg-query -W -f '${Version}' ceph 2024-03-07T02:52:23.401 INFO:teuthology.orchestra.run.smithi148.stdout:18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:52:23.402 INFO:teuthology.packaging:The installed version of ceph is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:52:23.402 INFO:teuthology.task.install:The correct ceph version 18.2.1-707-gffcfe7db-1jammy is installed. 2024-03-07T02:52:23.403 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-07T02:52:23.403 INFO:teuthology.packaging:ref: None 2024-03-07T02:52:23.403 INFO:teuthology.packaging:tag: None 2024-03-07T02:52:23.403 INFO:teuthology.packaging:branch: reef 2024-03-07T02:52:23.403 INFO:teuthology.packaging:sha1: ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:52:23.404 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-03-07T02:52:23.749 DEBUG:teuthology.orchestra.run.smithi195:> dpkg-query -W -f '${Version}' ceph 2024-03-07T02:52:23.769 INFO:teuthology.orchestra.run.smithi195.stdout:18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:52:23.769 INFO:teuthology.packaging:The installed version of ceph is 18.2.1-707-gffcfe7db-1jammy 2024-03-07T02:52:23.769 INFO:teuthology.task.install:The correct ceph version 18.2.1-707-gffcfe7db-1jammy is installed. 2024-03-07T02:52:23.770 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-07T02:52:23.770 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:52:23.770 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-07T02:52:23.785 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T02:52:23.785 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-07T02:52:23.823 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-07T02:52:23.824 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:52:23.824 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/usr/bin/daemon-helper 2024-03-07T02:52:23.842 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-07T02:52:23.897 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T02:52:23.897 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/daemon-helper 2024-03-07T02:52:23.910 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-07T02:52:23.961 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-07T02:52:23.961 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:52:23.962 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-07T02:52:23.974 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-07T02:52:24.029 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T02:52:24.029 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-07T02:52:24.041 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-07T02:52:24.094 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-07T02:52:24.094 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:52:24.094 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/usr/bin/stdin-killer 2024-03-07T02:52:24.108 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-07T02:52:24.161 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T02:52:24.161 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/usr/bin/stdin-killer 2024-03-07T02:52:24.174 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-07T02:52:24.226 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-07T02:52:24.310 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71'} 2024-03-07T02:52:24.310 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-03-07T02:52:24.311 INFO:tasks.cephadm:Cluster fsid is b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:52:24.311 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-07T02:52:24.311 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.148', 'mon.c': '[v2:172.21.15.148:3301,v1:172.21.15.148:6790]', 'mon.b': '172.21.15.195'} 2024-03-07T02:52:24.311 INFO:tasks.cephadm:First mon is mon.a on smithi148 2024-03-07T02:52:24.311 INFO:tasks.cephadm:First mgr is y 2024-03-07T02:52:24.311 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-07T02:52:24.311 DEBUG:teuthology.orchestra.run.smithi148:> sudo hostname $(hostname -s) 2024-03-07T02:52:24.326 DEBUG:teuthology.orchestra.run.smithi195:> sudo hostname $(hostname -s) 2024-03-07T02:52:24.339 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-03-07T02:52:24.339 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T02:52:24.553 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'ea9a5688d9980e4ddedb59679d2960bcbac6fc71', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=jammy,DIST=jammy,MACHINE_SIZE=gigantic/54541/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-1905-gea9a5688', 'node_name': '172.21.2.17+braggi17', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=jammy,DIST=jammy,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-1905-gea9a5688-1jammy'}, 'url': 'https://1.chacra.ceph.com/r/ceph/main/ea9a5688d9980e4ddedb59679d2960bcbac6fc71/ubuntu/jammy/flavors/default/', 'distro_codename': 'jammy', 'modified': '2024-03-06 23:55:50.104725', 'distro_version': '22.04', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://1.chacra.ceph.com/repos/ceph/main/ea9a5688d9980e4ddedb59679d2960bcbac6fc71/ubuntu/jammy/flavors/default/', 'archs': ['x86_64'], 'distro': 'ubuntu'}] 2024-03-07T02:52:24.854 INFO:tasks.util.chacra:got chacra host 3.chacra.ceph.com, ref reef, sha1 ffcfe7db8c5ae0acba06074778b2c315e0dca035 from https://shaman.ceph.com/api/search/?project=ceph&distros=ubuntu%2F22.04%2Fx86_64&flavor=default&ref=reef 2024-03-07T02:52:24.855 INFO:tasks.cephadm:Discovered cachra url: https://3.chacra.ceph.com/binaries/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/x86_64/flavors/default/cephadm 2024-03-07T02:52:24.855 DEBUG:teuthology.orchestra.run.smithi148:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-07T02:52:25.426 INFO:teuthology.orchestra.run.smithi148.stdout:-rw-rw-r-- 1 ubuntu ubuntu 216271 Mar 7 02:52 /home/ubuntu/cephtest/cephadm 2024-03-07T02:52:25.426 DEBUG:teuthology.orchestra.run.smithi195:> curl --silent -L https://3.chacra.ceph.com/binaries/ceph/reef/ffcfe7db8c5ae0acba06074778b2c315e0dca035/ubuntu/jammy/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-07T02:52:26.000 INFO:teuthology.orchestra.run.smithi195.stdout:-rw-rw-r-- 1 ubuntu ubuntu 216271 Mar 7 02:52 /home/ubuntu/cephtest/cephadm 2024-03-07T02:52:26.000 DEBUG:teuthology.orchestra.run.smithi148:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-07T02:52:26.007 DEBUG:teuthology.orchestra.run.smithi195:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-07T02:52:26.026 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-03-07T02:52:26.026 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-03-07T02:52:26.051 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-03-07T02:52:26.169 INFO:teuthology.orchestra.run.smithi148.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-03-07T02:52:26.178 INFO:teuthology.orchestra.run.smithi195.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-03-07T02:54:40.741 INFO:teuthology.orchestra.run.smithi148.stdout:{ 2024-03-07T02:54:40.741 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph_version": "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)", 2024-03-07T02:54:40.741 INFO:teuthology.orchestra.run.smithi148.stdout: "image_id": "0164bb8a7c6757b1a14e365a606a78ffe7cebb460f04a1ceb58f5dcf21071c24", 2024-03-07T02:54:40.742 INFO:teuthology.orchestra.run.smithi148.stdout: "repo_digests": [ 2024-03-07T02:54:40.742 INFO:teuthology.orchestra.run.smithi148.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4e788e3c6821efe986a710ffe10c052cd830c07b33cd4a66b97449fc9bbef8e7" 2024-03-07T02:54:40.742 INFO:teuthology.orchestra.run.smithi148.stdout: ] 2024-03-07T02:54:40.742 INFO:teuthology.orchestra.run.smithi148.stdout:} 2024-03-07T02:58:31.456 INFO:teuthology.orchestra.run.smithi195.stdout:{ 2024-03-07T02:58:31.457 INFO:teuthology.orchestra.run.smithi195.stdout: "ceph_version": "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)", 2024-03-07T02:58:31.457 INFO:teuthology.orchestra.run.smithi195.stdout: "image_id": "0164bb8a7c6757b1a14e365a606a78ffe7cebb460f04a1ceb58f5dcf21071c24", 2024-03-07T02:58:31.457 INFO:teuthology.orchestra.run.smithi195.stdout: "repo_digests": [ 2024-03-07T02:58:31.457 INFO:teuthology.orchestra.run.smithi195.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:4e788e3c6821efe986a710ffe10c052cd830c07b33cd4a66b97449fc9bbef8e7" 2024-03-07T02:58:31.457 INFO:teuthology.orchestra.run.smithi195.stdout: ] 2024-03-07T02:58:31.457 INFO:teuthology.orchestra.run.smithi195.stdout:} 2024-03-07T02:58:31.492 DEBUG:teuthology.orchestra.run.smithi148:> sudo mkdir -p /etc/ceph 2024-03-07T02:58:31.507 DEBUG:teuthology.orchestra.run.smithi195:> sudo mkdir -p /etc/ceph 2024-03-07T02:58:31.521 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod 777 /etc/ceph 2024-03-07T02:58:31.562 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 777 /etc/ceph 2024-03-07T02:58:31.580 INFO:tasks.cephadm:Writing seed config... 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-07T02:58:31.581 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-07T02:58:31.582 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-07T02:58:31.582 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-07T02:58:31.582 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-07T02:58:31.582 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:58:31.582 DEBUG:teuthology.orchestra.run.smithi148:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-07T02:58:31.607 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b9c09ac8-dc2d-11ee-95c6-87774f69a715 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-07T02:58:31.608 DEBUG:teuthology.orchestra.run.smithi148:mon.a> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a.service 2024-03-07T02:58:31.653 DEBUG:teuthology.orchestra.run.smithi148:mgr.y> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y.service 2024-03-07T02:58:31.696 INFO:tasks.cephadm:Bootstrapping... 2024-03-07T02:58:31.696 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.148 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-07T02:58:31.860 INFO:teuthology.orchestra.run.smithi148.stdout:-------------------------------------------------------------------------------- 2024-03-07T02:58:31.860 INFO:teuthology.orchestra.run.smithi148.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', 'b9c09ac8-dc2d-11ee-95c6-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.148', '--skip-admin-label'] 2024-03-07T02:58:31.860 INFO:teuthology.orchestra.run.smithi148.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-03-07T02:58:31.860 INFO:teuthology.orchestra.run.smithi148.stdout:Verifying podman|docker is present... 2024-03-07T02:58:31.860 INFO:teuthology.orchestra.run.smithi148.stdout:Verifying lvm2 is present... 2024-03-07T02:58:31.861 INFO:teuthology.orchestra.run.smithi148.stdout:Verifying time synchronization is in place... 2024-03-07T02:58:31.866 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-07T02:58:31.866 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-07T02:58:31.870 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-07T02:58:31.870 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.875 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-03-07T02:58:31.875 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-03-07T02:58:31.879 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-03-07T02:58:31.879 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.884 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-03-07T02:58:31.884 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout masked 2024-03-07T02:58:31.888 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-03-07T02:58:31.888 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.892 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-03-07T02:58:31.892 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-03-07T02:58:31.895 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-03-07T02:58:31.895 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.899 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout enabled 2024-03-07T02:58:31.903 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout active 2024-03-07T02:58:31.903 INFO:teuthology.orchestra.run.smithi148.stdout:Unit ntp.service is enabled and running 2024-03-07T02:58:31.903 INFO:teuthology.orchestra.run.smithi148.stdout:Repeating the final host check... 2024-03-07T02:58:31.903 INFO:teuthology.orchestra.run.smithi148.stdout:docker (/usr/bin/docker) is present 2024-03-07T02:58:31.903 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl is present 2024-03-07T02:58:31.903 INFO:teuthology.orchestra.run.smithi148.stdout:lvcreate is present 2024-03-07T02:58:31.906 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-07T02:58:31.906 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-07T02:58:31.909 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-07T02:58:31.910 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.913 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-03-07T02:58:31.913 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-03-07T02:58:31.916 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-03-07T02:58:31.916 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.919 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-03-07T02:58:31.919 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout masked 2024-03-07T02:58:31.922 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-03-07T02:58:31.923 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.926 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-03-07T02:58:31.926 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-03-07T02:58:31.929 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-03-07T02:58:31.929 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout inactive 2024-03-07T02:58:31.932 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout enabled 2024-03-07T02:58:31.936 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stdout active 2024-03-07T02:58:31.936 INFO:teuthology.orchestra.run.smithi148.stdout:Unit ntp.service is enabled and running 2024-03-07T02:58:31.936 INFO:teuthology.orchestra.run.smithi148.stdout:Host looks OK 2024-03-07T02:58:31.936 INFO:teuthology.orchestra.run.smithi148.stdout:Cluster fsid: b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:31.936 INFO:teuthology.orchestra.run.smithi148.stdout:Acquiring lock 139823830320512 on /run/cephadm/b9c09ac8-dc2d-11ee-95c6-87774f69a715.lock 2024-03-07T02:58:31.936 INFO:teuthology.orchestra.run.smithi148.stdout:Lock 139823830320512 acquired on /run/cephadm/b9c09ac8-dc2d-11ee-95c6-87774f69a715.lock 2024-03-07T02:58:31.937 INFO:teuthology.orchestra.run.smithi148.stdout:Verifying IP 172.21.15.148 port 3300 ... 2024-03-07T02:58:31.937 INFO:teuthology.orchestra.run.smithi148.stdout:Verifying IP 172.21.15.148 port 6789 ... 2024-03-07T02:58:31.937 INFO:teuthology.orchestra.run.smithi148.stdout:Base mon IP(s) is [172.21.15.148:3300, 172.21.15.148:6789], mon addrv is [v2:172.21.15.148:3300,v1:172.21.15.148:6789] 2024-03-07T02:58:31.939 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-03-07T02:58:31.939 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.148 metric 100 2024-03-07T02:58:31.940 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-03-07T02:58:31.940 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.148 metric 100 2024-03-07T02:58:31.940 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.148 metric 100 2024-03-07T02:58:31.940 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.148 metric 100 2024-03-07T02:58:31.940 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.148 metric 100 2024-03-07T02:58:31.941 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-03-07T02:58:31.941 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-07T02:58:31.941 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1767sec pref medium 2024-03-07T02:58:31.943 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-07T02:58:31.943 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-03-07T02:58:31.943 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-07T02:58:31.943 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-07T02:58:31.944 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:7257/64 scope link 2024-03-07T02:58:31.944 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.0.0/20` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.0.0/20` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.0.1/32` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.0.1/32` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.0.2/32` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.0.2/32` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.15.254/32` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Mon IP `172.21.15.148` is in CIDR network `172.21.15.254/32` 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-03-07T02:58:31.945 INFO:teuthology.orchestra.run.smithi148.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-07T02:58:31.946 INFO:teuthology.orchestra.run.smithi148.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-03-07T02:58:32.340 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/docker: stdout reef: Pulling from ceph-ci/ceph 2024-03-07T02:58:32.341 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/docker: stdout Digest: sha256:4e788e3c6821efe986a710ffe10c052cd830c07b33cd4a66b97449fc9bbef8e7 2024-03-07T02:58:32.341 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:reef 2024-03-07T02:58:32.341 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:reef 2024-03-07T02:58:33.597 INFO:teuthology.orchestra.run.smithi148.stdout:ceph: stdout ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable) 2024-03-07T02:58:33.597 INFO:teuthology.orchestra.run.smithi148.stdout:Ceph version: ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable) 2024-03-07T02:58:33.597 INFO:teuthology.orchestra.run.smithi148.stdout:Extracting ceph user uid/gid from container image... 2024-03-07T02:58:34.760 INFO:teuthology.orchestra.run.smithi148.stdout:stat: stdout 167 167 2024-03-07T02:58:34.760 INFO:teuthology.orchestra.run.smithi148.stdout:Creating initial keys... 2024-03-07T02:58:35.930 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-authtool: stdout AQBbLell+GGFGBAANw19HLGdLIGJOQvhcPz/1A== 2024-03-07T02:58:37.127 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-authtool: stdout AQBcLelld6LwJBAA0mBfOkhwmELAeu6pkA4HQQ== 2024-03-07T02:58:38.306 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-authtool: stdout AQBdLellFYu0LhAAyuFF9AGkBoNbG5jOIFsveg== 2024-03-07T02:58:38.307 INFO:teuthology.orchestra.run.smithi148.stdout:Creating initial monmap... 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:monmaptool for a [v2:172.21.15.148:3300,v1:172.21.15.148:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:setting min_mon_release = pacific 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/monmaptool: set fsid to b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-07T02:58:39.378 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:58:39.379 INFO:teuthology.orchestra.run.smithi148.stdout:Creating mon... 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.067+0000 7f4f0f036c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.067+0000 7f4f0f036c80 1 imported monmap: 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-03-07T02:58:38.930467+0000 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr created 2024-03-07T02:58:38.930467+0000 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr min_mon_release 16 (pacific) 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon.a 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.067+0000 7f4f0f036c80 0 /usr/bin/ceph-mon: set fsid to b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:40.569 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Git sha 0 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Compile date 2024-03-06 23:19:49 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: DB SUMMARY 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: DB Session ID: 1V911BPV9NT1WJEKXGKW 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.create_if_missing: 1 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T02:58:40.570 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.env: 0x563136725720 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.info_log: 0x563139506720 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.statistics: (nil) 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.use_fsync: 0 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T02:58:40.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.db_log_dir: 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.wal_dir: 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.write_buffer_manager: 0x563138797180 2024-03-07T02:58:40.572 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.unordered_write: 0 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.row_cache: None 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.wal_filter: None 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T02:58:40.573 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.wal_compression: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T02:58:40.574 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_open_files: -1 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Compression algorithms supported: 2024-03-07T02:58:40.575 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kZSTD supported: 0 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/db_impl/db_impl_open.cc:317] Creating manifest 1 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T02:58:40.576 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.merge_operator: 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_filter: None 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563139506840) 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-03-07T02:58:40.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr checksum: 4 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x5631388171f0 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-03-07T02:58:40.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr filter_policy: bloomfilter 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr format_version: 5 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr max_auto_readahead_size: 262144 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr prepopulate_block_cache: 0 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr initial_auto_readahead_size: 8192 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr num_file_reads_for_auto_readahead: 2 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression: NoCompression 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T02:58:40.579 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.num_levels: 7 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:58:40.580 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T02:58:40.581 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T02:58:40.582 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T02:58:40.583 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.ttl: 2592000 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T02:58:40.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8213d278-f7a6-4983-a9aa-c0583be7d69e 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 5 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.071+0000 7f4f0f036c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56313888e000 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.075+0000 7f4f0f036c80 4 rocksdb: DB pointer 0x563138880000 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.075+0000 7f4eff236700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.075+0000 7f4eff236700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:58:40.585 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.586 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Block cache BinnedLRUCache@0x5631388171f0#8 capacity: 512.00 MB usage: 0.00 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr Block cache entry stats(count,size,portion): Misc(1,0.00 KB,0%) 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.075+0000 7f4f0f036c80 4 rocksdb: [db/db_impl/db_impl.cc:496] Shutdown: canceling all background work 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.075+0000 7f4f0f036c80 4 rocksdb: [db/db_impl/db_impl.cc:704] Shutdown complete 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph-mon: stderr debug 2024-03-07T02:58:40.075+0000 7f4f0f036c80 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-03-07T02:58:40.587 INFO:teuthology.orchestra.run.smithi148.stdout:create mon.a on 2024-03-07T02:58:40.822 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-03-07T02:58:41.014 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-03-07T02:58:41.222 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715.target -> /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715.target. 2024-03-07T02:58:41.222 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715.target -> /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715.target. 2024-03-07T02:58:41.473 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a 2024-03-07T02:58:41.473 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a.service: Unit ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a.service not loaded. 2024-03-07T02:58:41.646 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715.target.wants/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a.service -> /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service. 2024-03-07T02:58:41.655 INFO:teuthology.orchestra.run.smithi148.stdout:firewalld does not appear to be present 2024-03-07T02:58:41.655 INFO:teuthology.orchestra.run.smithi148.stdout:Not possible to enable service . firewalld.service is not available 2024-03-07T02:58:41.656 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for mon to start... 2024-03-07T02:58:41.656 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for mon... 2024-03-07T02:58:43.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:42 smithi148 bash[20873]: cluster 2024-03-07T02:58:42.785335+0000 mon.a (mon.0) 1 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout cluster: 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout id: b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout services: 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.257653s) 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-03-07T02:58:43.577 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout data: 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout pgs: 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:mon is available 2024-03-07T02:58:43.578 INFO:teuthology.orchestra.run.smithi148.stdout:Assimilating anything we can from ceph.conf... 2024-03-07T02:58:44.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:43 smithi148 bash[20873]: cluster 2024-03-07T02:58:42.788571+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-03-07T02:58:44.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:43 smithi148 bash[20873]: cluster 2024-03-07T02:58:42.788783+0000 mon.a (mon.0) 3 : cluster 0 monmap e1: 1 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:44.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:43 smithi148 bash[20873]: cluster 2024-03-07T02:58:42.790870+0000 mon.a (mon.0) 4 : cluster 0 fsmap 2024-03-07T02:58:44.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:43 smithi148 bash[20873]: cluster 2024-03-07T02:58:42.793066+0000 mon.a (mon.0) 5 : cluster 0 osdmap e1: 0 total, 0 up, 0 in 2024-03-07T02:58:44.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:43 smithi148 bash[20873]: cluster 2024-03-07T02:58:42.793503+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e1: no daemons active 2024-03-07T02:58:44.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:43 smithi148 bash[20873]: audit 2024-03-07T02:58:43.046154+0000 mon.a (mon.0) 7 : audit 0 from='client.? 172.21.15.148:0/2414310706' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout [global] 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout fsid = b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.148:3300,v1:172.21.15.148:6789] 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:44.982 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-07T02:58:44.983 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-07T02:58:44.983 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:44.983 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout [osd] 2024-03-07T02:58:44.983 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-07T02:58:44.983 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-07T02:58:44.983 INFO:teuthology.orchestra.run.smithi148.stdout:Generating new minimal ceph.conf... 2024-03-07T02:58:45.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:44 smithi148 bash[20873]: audit 2024-03-07T02:58:44.423146+0000 mon.a (mon.0) 8 : audit 1 from='client.? 172.21.15.148:0/885400791' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-07T02:58:45.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:44 smithi148 bash[20873]: audit 2024-03-07T02:58:44.428763+0000 mon.a (mon.0) 9 : audit 1 from='client.? 172.21.15.148:0/885400791' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-07T02:58:46.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:45 smithi148 bash[20873]: audit 2024-03-07T02:58:45.889151+0000 mon.a (mon.0) 10 : audit 0 from='client.? 172.21.15.148:0/1910790265' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T02:58:46.394 INFO:teuthology.orchestra.run.smithi148.stdout:Restarting the monitor... 2024-03-07T02:58:46.717 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:46 smithi148 systemd[1]: Stopping Ceph mon.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T02:58:46.718 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:46 smithi148 bash[20873]: debug 2024-03-07T02:58:46.531+0000 7f868f3bc700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T02:58:46.718 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:46 smithi148 bash[20873]: debug 2024-03-07T02:58:46.531+0000 7f868f3bc700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-03-07T02:58:46.994 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:46 smithi148 bash[21260]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T02:58:47.010 INFO:teuthology.orchestra.run.smithi148.stdout:Setting public_network to 172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32 in mon config section 2024-03-07T02:58:47.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:47 smithi148 bash[21322]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T02:58:47.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:47 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-07T02:58:47.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:47 smithi148 systemd[1]: Stopped Ceph mon.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T02:58:47.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:47 smithi148 systemd[1]: Started Ceph mon.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:47.995+0000 7f752407bc80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:47.995+0000 7f752407bc80 0 ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable), process ceph-mon, pid 7 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:47.995+0000 7f752407bc80 0 pidfile_write: ignore empty --pid-file 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 0 load: jerasure load: lrc 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Git sha 0 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Compile date 2024-03-06 23:19:49 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: DB SUMMARY 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: DB Session ID: DQ0GHHO86HYH64Z4KKCD 2024-03-07T02:58:48.247 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84540 ; 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.env: 0x557c0ede5720 2024-03-07T02:58:48.248 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.info_log: 0x557c10639500 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.statistics: (nil) 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.use_fsync: 0 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T02:58:48.249 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.db_log_dir: 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.wal_dir: 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.write_buffer_manager: 0x557c0f8c6320 2024-03-07T02:58:48.250 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T02:58:48.251 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T02:58:48.251 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T02:58:48.251 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T02:58:48.251 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T02:58:48.251 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T02:58:48.252 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T02:58:48.252 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T02:58:48.252 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.unordered_write: 0 2024-03-07T02:58:48.252 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.row_cache: None 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.wal_filter: None 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.wal_compression: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T02:58:48.253 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T02:58:48.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_open_files: -1 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Compression algorithms supported: 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kZSTD supported: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T02:58:48.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.merge_operator: 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_filter: None 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T02:58:48.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x557c10639600) 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: cache_index_and_filter_blocks: 1 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: pin_top_level_index_and_filter: 1 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: index_type: 0 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: data_block_index_type: 0 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: index_shortening: 1 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: checksum: 4 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: no_block_cache: 0 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_cache: 0x557c0f9491f0 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_cache_name: BinnedLRUCache 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_cache_options: 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: capacity : 536870912 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: num_shard_bits : 4 2024-03-07T02:58:48.257 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: strict_capacity_limit : 0 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: high_pri_pool_ratio: 0.000 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_cache_compressed: (nil) 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: persistent_cache: (nil) 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_size: 4096 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_size_deviation: 10 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_restart_interval: 16 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: index_block_restart_interval: 1 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: metadata_block_size: 4096 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: partition_filters: 0 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: use_delta_encoding: 1 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: filter_policy: bloomfilter 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: whole_key_filtering: 1 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: verify_compression: 0 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: read_amp_bytes_per_bit: 0 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: format_version: 5 2024-03-07T02:58:48.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: enable_index_compression: 1 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: block_align: 0 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: max_auto_readahead_size: 262144 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: prepopulate_block_cache: 0 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: initial_auto_readahead_size: 8192 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: num_file_reads_for_auto_readahead: 2 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression: NoCompression 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.num_levels: 7 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T02:58:48.259 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T02:58:48.260 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T02:58:48.261 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T02:58:48.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.ttl: 2592000 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T02:58:48.263 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8213d278-f7a6-4983-a9aa-c0583be7d69e 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780328006864, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780328008034, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81064, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79240, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73441, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709780328, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8213d278-f7a6-4983-a9aa-c0583be7d69e", "db_session_id": "DQ0GHHO86HYH64Z4KKCD", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-03-07T02:58:48.264 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.003+0000 7f752407bc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780328008136, "job": 1, "event": "recovery_finished"} 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x557c0f9ea000 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 4 rocksdb: DB pointer 0x557c0f9d8000 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f7512875700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f7512875700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: ** DB Stats ** 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: ** Compaction Stats [default] ** 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:58:48.265 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: L0 2/0 81.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Sum 2/0 81.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 112.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: ** Compaction Stats [default] ** 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 112.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Flush(GB): cumulative 0.000, interval 0.000 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Cumulative compaction: 0.00 GB write, 18.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Interval compaction: 0.00 GB write, 18.33 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T02:58:48.266 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Block cache BinnedLRUCache@0x557c0f9491f0#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.4e-05 secs_since: 0 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] at bind addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 1 mon.a@-1(???) e1 preinit fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 0 mon.a@-1(???).mds e1 new map 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 0 mon.a@-1(???).mds e1 print_map 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: e1 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: legacy client fscid: -1 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: No filesystems configured 2024-03-07T02:58:48.267 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: debug 2024-03-07T02:58:48.007+0000 7f752407bc80 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: cluster 2024-03-07T02:58:48.032689+0000 mon.a (mon.0) 1 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: cluster 2024-03-07T02:58:48.032752+0000 mon.a (mon.0) 2 : cluster 0 monmap e1: 1 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: cluster 2024-03-07T02:58:48.033161+0000 mon.a (mon.0) 3 : cluster 0 fsmap 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: cluster 2024-03-07T02:58:48.033192+0000 mon.a (mon.0) 4 : cluster 0 osdmap e1: 0 total, 0 up, 0 in 2024-03-07T02:58:48.268 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 bash[21362]: cluster 2024-03-07T02:58:48.033603+0000 mon.a (mon.0) 5 : cluster 0 mgrmap e1: no daemons active 2024-03-07T02:58:48.762 INFO:teuthology.orchestra.run.smithi148.stdout:Wrote config to /etc/ceph/ceph.conf 2024-03-07T02:58:48.788 INFO:teuthology.orchestra.run.smithi148.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-07T02:58:48.788 INFO:teuthology.orchestra.run.smithi148.stdout:Creating mgr... 2024-03-07T02:58:48.788 INFO:teuthology.orchestra.run.smithi148.stdout:Verifying port 0.0.0.0:9283 ... 2024-03-07T02:58:48.789 INFO:teuthology.orchestra.run.smithi148.stdout:Verifying port 0.0.0.0:8765 ... 2024-03-07T02:58:49.097 INFO:teuthology.orchestra.run.smithi148.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y 2024-03-07T02:58:49.097 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y.service: Unit ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y.service not loaded. 2024-03-07T02:58:49.209 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:49.291 INFO:teuthology.orchestra.run.smithi148.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715.target.wants/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y.service -> /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service. 2024-03-07T02:58:49.298 INFO:teuthology.orchestra.run.smithi148.stdout:firewalld does not appear to be present 2024-03-07T02:58:49.298 INFO:teuthology.orchestra.run.smithi148.stdout:Not possible to enable service . firewalld.service is not available 2024-03-07T02:58:49.298 INFO:teuthology.orchestra.run.smithi148.stdout:firewalld does not appear to be present 2024-03-07T02:58:49.298 INFO:teuthology.orchestra.run.smithi148.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-03-07T02:58:49.298 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for mgr to start... 2024-03-07T02:58:49.298 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for mgr... 2024-03-07T02:58:49.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T02:58:49.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:49 smithi148 bash[21362]: audit 2024-03-07T02:58:48.234070+0000 mon.a (mon.0) 6 : audit 1 from='client.? 172.21.15.148:0/250936257' entity='client.admin' 2024-03-07T02:58:51.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:50 smithi148 bash[21362]: audit 2024-03-07T02:58:50.706370+0000 mon.a (mon.0) 7 : audit 0 from='client.? 172.21.15.148:0/4020433490' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:58:51.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:50 smithi148 bash[21615]: debug 2024-03-07T02:58:50.687+0000 7f579b51d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsid": "b9c09ac8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:58:51.215 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:58:51.216 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:58:51.220 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:51.221 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:58:42.789924+0000", 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:58:51.222 INFO:teuthology.orchestra.run.smithi148.stdout:mgr not available, waiting (1/15)... 2024-03-07T02:58:52.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:52 smithi148 bash[21615]: debug 2024-03-07T02:58:52.239+0000 7f579b51d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T02:58:52.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:52 smithi148 bash[21615]: debug 2024-03-07T02:58:52.335+0000 7f579b51d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T02:58:53.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:52 smithi148 bash[21615]: debug 2024-03-07T02:58:52.579+0000 7f579b51d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T02:58:53.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:53 smithi148 bash[21615]: debug 2024-03-07T02:58:53.355+0000 7f579b51d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T02:58:53.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:53 smithi148 bash[21615]: debug 2024-03-07T02:58:53.463+0000 7f579b51d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T02:58:53.875 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:53 smithi148 bash[21615]: debug 2024-03-07T02:58:53.563+0000 7f579b51d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T02:58:54.138 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:53 smithi148 bash[21615]: debug 2024-03-07T02:58:53.919+0000 7f579b51d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T02:58:54.138 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:54 smithi148 bash[21615]: debug 2024-03-07T02:58:54.031+0000 7f579b51d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T02:58:54.138 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:54 smithi148 bash[21615]: debug 2024-03-07T02:58:54.135+0000 7f579b51d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T02:58:54.488 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:54 smithi148 bash[21362]: audit 2024-03-07T02:58:54.197130+0000 mon.a (mon.0) 8 : audit 0 from='client.? 172.21.15.148:0/1289071237' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:58:54.711 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:54.711 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:58:54.711 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsid": "b9c09ac8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:58:54.711 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:54.712 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:58:54.713 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:58:54.716 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:58:54.716 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:58:54.716 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:58:54.717 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:58:42.789924+0000", 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:58:54.718 INFO:teuthology.orchestra.run.smithi148.stdout:mgr not available, waiting (2/15)... 2024-03-07T02:58:54.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:54 smithi148 bash[21615]: debug 2024-03-07T02:58:54.487+0000 7f579b51d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T02:58:54.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:54 smithi148 bash[21615]: debug 2024-03-07T02:58:54.727+0000 7f579b51d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T02:58:55.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:54 smithi148 bash[21615]: debug 2024-03-07T02:58:54.979+0000 7f579b51d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T02:58:55.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:55 smithi148 bash[21615]: debug 2024-03-07T02:58:55.087+0000 7f579b51d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T02:58:55.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:55 smithi148 bash[21615]: debug 2024-03-07T02:58:55.355+0000 7f579b51d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T02:58:55.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:55 smithi148 bash[21615]: debug 2024-03-07T02:58:55.591+0000 7f579b51d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:58:56.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:55 smithi148 bash[21615]: debug 2024-03-07T02:58:55.987+0000 7f579b51d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T02:58:56.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:56 smithi148 bash[21615]: debug 2024-03-07T02:58:56.103+0000 7f579b51d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T02:58:56.735 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:56 smithi148 bash[21615]: debug 2024-03-07T02:58:56.347+0000 7f579b51d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:58:57.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:57 smithi148 bash[21615]: debug 2024-03-07T02:58:57.347+0000 7f579b51d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T02:58:57.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:57 smithi148 bash[21615]: debug 2024-03-07T02:58:57.463+0000 7f579b51d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T02:58:58.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:57 smithi148 bash[21362]: audit 2024-03-07T02:58:57.615413+0000 mon.a (mon.0) 9 : audit 0 from='client.? 172.21.15.148:0/2547484346' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:58:58.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:57 smithi148 bash[21615]: debug 2024-03-07T02:58:57.755+0000 7f579b51d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T02:58:58.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:57 smithi148 bash[21615]: debug 2024-03-07T02:58:57.871+0000 7f579b51d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T02:58:58.168 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:58:58.168 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:58:58.168 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsid": "b9c09ac8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:58:58.168 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:58:58.169 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:58:58.170 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:58:58.171 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:58:42.789924+0000", 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:58:58.172 INFO:teuthology.orchestra.run.smithi148.stdout:mgr not available, waiting (3/15)... 2024-03-07T02:58:58.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:58 smithi148 bash[21615]: debug 2024-03-07T02:58:58.315+0000 7f579b51d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T02:58:58.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:58:58 smithi148 bash[21615]: debug 2024-03-07T02:58:58.431+0000 7f579b51d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T02:58:59.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: cluster 2024-03-07T02:58:58.458639+0000 mon.a (mon.0) 10 : cluster 1 Activating manager daemon y 2024-03-07T02:58:59.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: cluster 2024-03-07T02:58:58.466027+0000 mon.a (mon.0) 11 : cluster 0 mgrmap e2: y(active, starting, since 0.00790484s) 2024-03-07T02:58:59.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.467098+0000 mon.a (mon.0) 12 : audit 0 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.467294+0000 mon.a (mon.0) 13 : audit 0 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.467479+0000 mon.a (mon.0) 14 : audit 0 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.468075+0000 mon.a (mon.0) 15 : audit 0 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.469123+0000 mon.a (mon.0) 16 : audit 0 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: cluster 2024-03-07T02:58:58.494197+0000 mon.a (mon.0) 17 : cluster 1 Manager daemon y is now available 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.507977+0000 mon.a (mon.0) 18 : audit 1 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.511076+0000 mon.a (mon.0) 19 : audit 1 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.514322+0000 mon.a (mon.0) 20 : audit 1 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.516282+0000 mon.a (mon.0) 21 : audit 1 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' 2024-03-07T02:58:59.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:58:59 smithi148 bash[21362]: audit 2024-03-07T02:58:58.518535+0000 mon.a (mon.0) 22 : audit 1 from='mgr.14100 172.21.15.148:0/4110615348' entity='mgr.y' 2024-03-07T02:59:00.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:00 smithi148 bash[21362]: cluster 2024-03-07T02:58:59.474338+0000 mon.a (mon.0) 23 : cluster 0 mgrmap e3: y(active, since 1.01621s) 2024-03-07T02:59:01.973 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:59:01.973 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:59:01.973 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsid": "b9c09ac8-dc2d-11ee-95c6-87774f69a715", 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "health": { 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 0 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "a" 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "quorum_age": 13, 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:59:01.974 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-07T02:59:01.975 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-07T02:59:01.978 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-07T02:59:01.979 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "restful" 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ], 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-07T02:59:01.980 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "modified": "2024-03-07T02:58:42.789924+0000", 2024-03-07T02:59:01.981 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-07T02:59:01.981 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout }, 2024-03-07T02:59:01.981 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-07T02:59:01.981 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:59:01.981 INFO:teuthology.orchestra.run.smithi148.stdout:mgr is available 2024-03-07T02:59:02.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:02 smithi148 bash[21362]: cluster 2024-03-07T02:59:01.471648+0000 mon.a (mon.0) 24 : cluster 0 mgrmap e4: y(active, since 3s) 2024-03-07T02:59:02.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:02 smithi148 bash[21362]: audit 2024-03-07T02:59:01.486577+0000 mon.a (mon.0) 25 : audit 0 from='client.? 172.21.15.148:0/4292763896' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-07T02:59:03.662 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:59:03.662 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout [global] 2024-03-07T02:59:03.662 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout fsid = b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:59:03.662 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-07T02:59:03.662 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-07T02:59:03.662 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-07T02:59:03.663 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-07T02:59:03.663 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:59:03.663 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-07T02:59:03.663 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-07T02:59:03.664 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 2024-03-07T02:59:03.664 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout [osd] 2024-03-07T02:59:03.664 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-07T02:59:03.664 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-07T02:59:03.664 INFO:teuthology.orchestra.run.smithi148.stdout:Enabling cephadm module... 2024-03-07T02:59:04.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:04 smithi148 bash[21362]: audit 2024-03-07T02:59:03.146348+0000 mon.a (mon.0) 26 : audit 1 from='client.? 172.21.15.148:0/1491726680' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-07T02:59:05.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:05 smithi148 bash[21362]: audit 2024-03-07T02:59:04.889797+0000 mon.a (mon.0) 27 : audit 1 from='client.? 172.21.15.148:0/449139125' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-07T02:59:05.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:05 smithi148 bash[21615]: ignoring --setuser ceph since I am not root 2024-03-07T02:59:05.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:05 smithi148 bash[21615]: ignoring --setgroup ceph since I am not root 2024-03-07T02:59:05.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:05 smithi148 bash[21615]: debug 2024-03-07T02:59:05.211+0000 7f0181360200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T02:59:06.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:06 smithi148 bash[21362]: audit 2024-03-07T02:59:05.031727+0000 mon.a (mon.0) 28 : audit 1 from='client.? 172.21.15.148:0/449139125' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-07T02:59:06.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:06 smithi148 bash[21362]: cluster 2024-03-07T02:59:05.031905+0000 mon.a (mon.0) 29 : cluster 0 mgrmap e5: y(active, since 6s) 2024-03-07T02:59:07.157 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:07 smithi148 bash[21362]: audit 2024-03-07T02:59:06.853437+0000 mon.a (mon.0) 30 : audit 0 from='client.? 172.21.15.148:0/3971489992' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-07T02:59:07.158 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:06 smithi148 bash[21615]: debug 2024-03-07T02:59:06.807+0000 7f0181360200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T02:59:07.158 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:06 smithi148 bash[21615]: debug 2024-03-07T02:59:06.907+0000 7f0181360200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for the mgr to restart... 2024-03-07T02:59:07.388 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for mgr epoch 5... 2024-03-07T02:59:07.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:07 smithi148 bash[21615]: debug 2024-03-07T02:59:07.155+0000 7f0181360200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T02:59:08.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:07 smithi148 bash[21615]: debug 2024-03-07T02:59:07.939+0000 7f0181360200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T02:59:08.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:08 smithi148 bash[21615]: debug 2024-03-07T02:59:08.051+0000 7f0181360200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T02:59:08.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:08 smithi148 bash[21615]: debug 2024-03-07T02:59:08.155+0000 7f0181360200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T02:59:08.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:08 smithi148 bash[21615]: debug 2024-03-07T02:59:08.503+0000 7f0181360200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T02:59:08.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:08 smithi148 bash[21615]: debug 2024-03-07T02:59:08.615+0000 7f0181360200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T02:59:08.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:08 smithi148 bash[21615]: debug 2024-03-07T02:59:08.719+0000 7f0181360200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T02:59:09.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:09 smithi148 bash[21615]: debug 2024-03-07T02:59:09.067+0000 7f0181360200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T02:59:09.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:09 smithi148 bash[21615]: debug 2024-03-07T02:59:09.303+0000 7f0181360200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T02:59:09.944 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:09 smithi148 bash[21615]: debug 2024-03-07T02:59:09.559+0000 7f0181360200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T02:59:09.944 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:09 smithi148 bash[21615]: debug 2024-03-07T02:59:09.671+0000 7f0181360200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T02:59:10.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:09 smithi148 bash[21615]: debug 2024-03-07T02:59:09.943+0000 7f0181360200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T02:59:10.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:10 smithi148 bash[21615]: debug 2024-03-07T02:59:10.183+0000 7f0181360200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:59:10.949 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:10 smithi148 bash[21615]: debug 2024-03-07T02:59:10.587+0000 7f0181360200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T02:59:10.949 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:10 smithi148 bash[21615]: debug 2024-03-07T02:59:10.703+0000 7f0181360200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T02:59:10.949 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:10 smithi148 bash[21615]: debug 2024-03-07T02:59:10.947+0000 7f0181360200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:59:12.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:11 smithi148 bash[21615]: debug 2024-03-07T02:59:11.935+0000 7f0181360200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T02:59:12.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:12 smithi148 bash[21615]: debug 2024-03-07T02:59:12.051+0000 7f0181360200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T02:59:12.802 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:12 smithi148 bash[21615]: debug 2024-03-07T02:59:12.335+0000 7f0181360200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T02:59:12.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:12 smithi148 bash[21615]: debug 2024-03-07T02:59:12.447+0000 7f0181360200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T02:59:13.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:12 smithi148 bash[21615]: debug 2024-03-07T02:59:12.891+0000 7f0181360200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T02:59:13.303 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:13 smithi148 bash[21615]: debug 2024-03-07T02:59:13.015+0000 7f0181360200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T02:59:14.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: cluster 2024-03-07T02:59:13.038324+0000 mon.a (mon.0) 31 : cluster 1 Active manager daemon y restarted 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: cluster 2024-03-07T02:59:13.038599+0000 mon.a (mon.0) 32 : cluster 1 Activating manager daemon y 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: cluster 2024-03-07T02:59:13.042541+0000 mon.a (mon.0) 33 : cluster 0 osdmap e2: 0 total, 0 up, 0 in 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: cluster 2024-03-07T02:59:13.042677+0000 mon.a (mon.0) 34 : cluster 0 mgrmap e6: y(active, starting, since 0.00418895s) 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.044740+0000 mon.a (mon.0) 35 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.046074+0000 mon.a (mon.0) 36 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.047249+0000 mon.a (mon.0) 37 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.047425+0000 mon.a (mon.0) 38 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.047584+0000 mon.a (mon.0) 39 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: cluster 2024-03-07T02:59:13.074448+0000 mon.a (mon.0) 40 : cluster 1 Manager daemon y is now available 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.092201+0000 mon.a (mon.0) 41 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.096048+0000 mon.a (mon.0) 42 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.100161+0000 mon.a (mon.0) 43 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.101532+0000 mon.a (mon.0) 44 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.105359+0000 mon.a (mon.0) 45 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:14.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.114909+0000 mon.a (mon.0) 46 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T02:59:14.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.118287+0000 mon.a (mon.0) 47 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T02:59:14.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.421977+0000 mon.a (mon.0) 48 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:14.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:14 smithi148 bash[21362]: audit 2024-03-07T02:59:13.426785+0000 mon.a (mon.0) 49 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:14.668 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:59:14.669 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-03-07T02:59:14.669 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-07T02:59:14.669 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:59:14.669 INFO:teuthology.orchestra.run.smithi148.stdout:mgr epoch 5 is available 2024-03-07T02:59:14.669 INFO:teuthology.orchestra.run.smithi148.stdout:Setting orchestrator backend to cephadm... 2024-03-07T02:59:15.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:15 smithi148 bash[21362]: cephadm 2024-03-07T02:59:13.089103+0000 mgr.y (mgr.14120) 1 : cephadm 1 Found migration_current of "None". Setting to last migration. 2024-03-07T02:59:15.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:15 smithi148 bash[21362]: audit 2024-03-07T02:59:14.048013+0000 mgr.y (mgr.14120) 2 : audit 0 from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-07T02:59:15.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:15 smithi148 bash[21362]: cluster 2024-03-07T02:59:14.049096+0000 mon.a (mon.0) 50 : cluster 0 mgrmap e7: y(active, since 1.01059s) 2024-03-07T02:59:15.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:15 smithi148 bash[21362]: audit 2024-03-07T02:59:14.059886+0000 mgr.y (mgr.14120) 3 : audit 0 from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-07T02:59:15.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:15 smithi148 bash[21362]: audit 2024-03-07T02:59:14.726299+0000 mon.a (mon.0) 51 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:16.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:16 smithi148 bash[21362]: cephadm 2024-03-07T02:59:14.458509+0000 mgr.y (mgr.14120) 4 : cephadm 1 [07/Mar/2024:02:59:14] ENGINE Bus STARTING 2024-03-07T02:59:16.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:16 smithi148 bash[21362]: cephadm 2024-03-07T02:59:14.561484+0000 mgr.y (mgr.14120) 5 : cephadm 1 [07/Mar/2024:02:59:14] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T02:59:16.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:16 smithi148 bash[21362]: cephadm 2024-03-07T02:59:14.678536+0000 mgr.y (mgr.14120) 6 : cephadm 1 [07/Mar/2024:02:59:14] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T02:59:16.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:16 smithi148 bash[21362]: cephadm 2024-03-07T02:59:14.678653+0000 mgr.y (mgr.14120) 7 : cephadm 1 [07/Mar/2024:02:59:14] ENGINE Bus STARTED 2024-03-07T02:59:16.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:16 smithi148 bash[21362]: cluster 2024-03-07T02:59:15.048346+0000 mon.a (mon.0) 52 : cluster 0 mgrmap e8: y(active, since 2s) 2024-03-07T02:59:16.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:16 smithi148 bash[21362]: audit 2024-03-07T02:59:15.864497+0000 mon.a (mon.0) 53 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:16.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:16 smithi148 bash[21362]: audit 2024-03-07T02:59:15.895210+0000 mon.a (mon.0) 54 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:17.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:17 smithi148 bash[21362]: audit 2024-03-07T02:59:15.860140+0000 mgr.y (mgr.14120) 8 : audit 0 from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:17.970 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout value unchanged 2024-03-07T02:59:17.970 INFO:teuthology.orchestra.run.smithi148.stdout:Generating ssh key... 2024-03-07T02:59:19.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21362]: audit 2024-03-07T02:59:17.522868+0000 mgr.y (mgr.14120) 9 : audit 0 from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: Generating public/private rsa key pair. 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: Your identification has been saved in /tmp/tmphnvsu8zz/key. 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: Your public key has been saved in /tmp/tmphnvsu8zz/key.pub. 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: The key fingerprint is: 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: SHA256:/sR6ykUyKAPOy17G7WCEope8/EK5p+pU3z0xjmPlFYA ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: The key's randomart image is: 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: +---[RSA 3072]----+ 2024-03-07T02:59:20.053 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: | .. | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: | E . | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: | . . | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: | o o . . | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: |. +o+ . S+.. | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: |.++=.+..*=+ | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: |.o*.*..=.=+ | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: |.+o=.o...=o | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: |oo==. . +o. | 2024-03-07T02:59:20.054 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:19 smithi148 bash[21615]: +----[SHA256]-----+ 2024-03-07T02:59:21.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:20 smithi148 bash[21362]: audit 2024-03-07T02:59:19.152617+0000 mgr.y (mgr.14120) 10 : audit 0 from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:21.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:20 smithi148 bash[21362]: cephadm 2024-03-07T02:59:19.153014+0000 mgr.y (mgr.14120) 11 : cephadm 1 Generating ssh key... 2024-03-07T02:59:21.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:20 smithi148 bash[21362]: audit 2024-03-07T02:59:19.631805+0000 mon.a (mon.0) 55 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:21.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:20 smithi148 bash[21362]: audit 2024-03-07T02:59:19.637501+0000 mon.a (mon.0) 56 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:21.697 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDN6MeDFLa253CKGwQWsXKSzwp32bspl4sUaeZ3WWAs/y3ODnV/eFi2MgQpjhSRyo5kvi+lQvrxkvZjKOqsR/hBVbHzY6TCjANmeaMP7tW2MTclpNBO21oVHTZ6ociGrdfdyaQ4FmFf4DAYGyGRJiHFSisgNTTz3kUjFgVnLsfZ7eIwAngLsLWJHyLdmbe4Zde3oEsm0SQFJVgH8edm3w4fCL13KtNXPDx45Sk73f1FkihEUCQQWnt50EAhYuDNoyTxBN1Ptkpun+5YHhfJAsvtBiHgEh8zyJnvdfhW15giyk/Ua3tosyD9BprStW4rRmL0cmbqNfk4sDGuCVdu9mFzg5GUzHWO22Luqu+AKHTosVKS3YO7QN543LF8VCYEx7Acxy3ylJ5Ef2jaxDBaQcSKw5sPkCQXseOE3x+qtzJ1On1jD8ZPqj4CHpZUVfO9lk0Cn5wsUA30xm9P7CboWBJoOrUvwf4FxfxgOsEYdWJaZzt9Wd5RO9qaAa0hrNBaey0= ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:59:21.697 INFO:teuthology.orchestra.run.smithi148.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-07T02:59:21.697 INFO:teuthology.orchestra.run.smithi148.stdout:Adding key to root@localhost authorized_keys... 2024-03-07T02:59:21.698 INFO:teuthology.orchestra.run.smithi148.stdout:Adding host smithi148... 2024-03-07T02:59:22.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:22 smithi148 bash[21362]: audit 2024-03-07T02:59:21.287610+0000 mgr.y (mgr.14120) 12 : audit 0 from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:24.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:24 smithi148 bash[21362]: audit 2024-03-07T02:59:22.809826+0000 mgr.y (mgr.14120) 13 : audit 0 from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi148", "addr": "172.21.15.148", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:25.280 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:25 smithi148 bash[21362]: cephadm 2024-03-07T02:59:23.365849+0000 mgr.y (mgr.14120) 14 : cephadm 1 Deploying cephadm binary to smithi148 2024-03-07T02:59:26.470 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout Added host 'smithi148' with addr '172.21.15.148' 2024-03-07T02:59:26.471 INFO:teuthology.orchestra.run.smithi148.stdout:Deploying unmanaged mon service... 2024-03-07T02:59:27.000 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:26 smithi148 bash[21362]: audit 2024-03-07T02:59:25.698571+0000 mon.a (mon.0) 57 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:27.000 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:26 smithi148 bash[21362]: cephadm 2024-03-07T02:59:25.699242+0000 mgr.y (mgr.14120) 15 : cephadm 1 Added host smithi148 2024-03-07T02:59:27.000 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:26 smithi148 bash[21362]: audit 2024-03-07T02:59:25.746191+0000 mon.a (mon.0) 58 : audit 0 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:28.287 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-03-07T02:59:28.287 INFO:teuthology.orchestra.run.smithi148.stdout:Deploying unmanaged mgr service... 2024-03-07T02:59:29.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:28 smithi148 bash[21362]: audit 2024-03-07T02:59:27.786023+0000 mgr.y (mgr.14120) 16 : audit 0 from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:29.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:28 smithi148 bash[21362]: cephadm 2024-03-07T02:59:27.789169+0000 mgr.y (mgr.14120) 17 : cephadm 1 Saving service mon spec with placement count:5 2024-03-07T02:59:29.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:28 smithi148 bash[21362]: audit 2024-03-07T02:59:27.794584+0000 mon.a (mon.0) 59 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:29.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:28 smithi148 bash[21362]: audit 2024-03-07T02:59:27.871504+0000 mon.a (mon.0) 60 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:29.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:28 smithi148 bash[21362]: audit 2024-03-07T02:59:28.197676+0000 mon.a (mon.0) 61 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:30.025 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-03-07T02:59:30.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:30 smithi148 bash[21362]: audit 2024-03-07T02:59:29.461977+0000 mgr.y (mgr.14120) 18 : audit 0 from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:30.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:30 smithi148 bash[21362]: cephadm 2024-03-07T02:59:29.464474+0000 mgr.y (mgr.14120) 19 : cephadm 1 Saving service mgr spec with placement count:2 2024-03-07T02:59:30.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:30 smithi148 bash[21362]: audit 2024-03-07T02:59:29.469489+0000 mon.a (mon.0) 62 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:32.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:32 smithi148 bash[21362]: audit 2024-03-07T02:59:31.214551+0000 mon.a (mon.0) 63 : audit 1 from='client.? 172.21.15.148:0/1376237716' entity='client.admin' 2024-03-07T02:59:33.353 INFO:teuthology.orchestra.run.smithi148.stdout:Enabling the dashboard module... 2024-03-07T02:59:34.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:33 smithi148 bash[21362]: audit 2024-03-07T02:59:32.814009+0000 mon.a (mon.0) 64 : audit 1 from='client.? 172.21.15.148:0/2062448382' entity='client.admin' 2024-03-07T02:59:34.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:33 smithi148 bash[21362]: cluster 2024-03-07T02:59:33.048709+0000 mgr.y (mgr.14120) 20 : cluster 0 pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:35.555 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:35 smithi148 bash[21362]: audit 2024-03-07T02:59:34.083581+0000 mon.a (mon.0) 65 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:35.555 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:35 smithi148 bash[21362]: audit 2024-03-07T02:59:34.631931+0000 mon.a (mon.0) 66 : audit 1 from='client.? 172.21.15.148:0/3243337895' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-07T02:59:35.555 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:35 smithi148 bash[21362]: audit 2024-03-07T02:59:34.768988+0000 mon.a (mon.0) 67 : audit 1 from='mgr.14120 172.21.15.148:0/1350969958' entity='mgr.y' 2024-03-07T02:59:35.556 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:35 smithi148 bash[21615]: ignoring --setuser ceph since I am not root 2024-03-07T02:59:35.556 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:35 smithi148 bash[21615]: ignoring --setgroup ceph since I am not root 2024-03-07T02:59:35.556 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:35 smithi148 bash[21615]: debug 2024-03-07T02:59:35.278+0000 7f56fbbe1200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T02:59:36.363 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:36 smithi148 bash[21362]: cluster 2024-03-07T02:59:35.049156+0000 mgr.y (mgr.14120) 21 : cluster 0 pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T02:59:36.363 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:36 smithi148 bash[21362]: audit 2024-03-07T02:59:35.097585+0000 mon.a (mon.0) 68 : audit 1 from='client.? 172.21.15.148:0/3243337895' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-07T02:59:36.363 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:36 smithi148 bash[21362]: cluster 2024-03-07T02:59:35.097693+0000 mon.a (mon.0) 69 : cluster 0 mgrmap e9: y(active, since 22s) 2024-03-07T02:59:37.192 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:36 smithi148 bash[21615]: debug 2024-03-07T02:59:36.838+0000 7f56fbbe1200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T02:59:37.192 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:36 smithi148 bash[21615]: debug 2024-03-07T02:59:36.938+0000 7f56fbbe1200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T02:59:37.192 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:37 smithi148 bash[21362]: audit 2024-03-07T02:59:36.997100+0000 mon.a (mon.0) 70 : audit 0 from='client.? 172.21.15.148:0/3949124514' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-07T02:59:37.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:37 smithi148 bash[21615]: debug 2024-03-07T02:59:37.190+0000 7f56fbbe1200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for the mgr to restart... 2024-03-07T02:59:37.556 INFO:teuthology.orchestra.run.smithi148.stdout:Waiting for mgr epoch 9... 2024-03-07T02:59:38.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:37 smithi148 bash[21615]: debug 2024-03-07T02:59:37.978+0000 7f56fbbe1200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T02:59:38.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:38 smithi148 bash[21615]: debug 2024-03-07T02:59:38.090+0000 7f56fbbe1200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T02:59:38.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:38 smithi148 bash[21615]: debug 2024-03-07T02:59:38.198+0000 7f56fbbe1200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T02:59:39.052 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:38 smithi148 bash[21615]: debug 2024-03-07T02:59:38.554+0000 7f56fbbe1200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T02:59:39.052 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:38 smithi148 bash[21615]: debug 2024-03-07T02:59:38.666+0000 7f56fbbe1200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T02:59:39.052 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:38 smithi148 bash[21615]: debug 2024-03-07T02:59:38.770+0000 7f56fbbe1200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T02:59:39.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:39 smithi148 bash[21615]: debug 2024-03-07T02:59:39.122+0000 7f56fbbe1200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T02:59:39.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:39 smithi148 bash[21615]: debug 2024-03-07T02:59:39.362+0000 7f56fbbe1200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T02:59:40.007 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:39 smithi148 bash[21615]: debug 2024-03-07T02:59:39.622+0000 7f56fbbe1200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T02:59:40.007 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:39 smithi148 bash[21615]: debug 2024-03-07T02:59:39.734+0000 7f56fbbe1200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T02:59:40.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:40 smithi148 bash[21615]: debug 2024-03-07T02:59:40.002+0000 7f56fbbe1200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T02:59:40.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:40 smithi148 bash[21615]: debug 2024-03-07T02:59:40.246+0000 7f56fbbe1200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:59:41.013 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:40 smithi148 bash[21615]: debug 2024-03-07T02:59:40.650+0000 7f56fbbe1200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T02:59:41.013 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:40 smithi148 bash[21615]: debug 2024-03-07T02:59:40.766+0000 7f56fbbe1200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T02:59:41.650 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:41 smithi148 bash[21615]: debug 2024-03-07T02:59:41.010+0000 7f56fbbe1200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T02:59:42.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:42 smithi148 bash[21615]: debug 2024-03-07T02:59:42.022+0000 7f56fbbe1200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T02:59:42.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:42 smithi148 bash[21615]: debug 2024-03-07T02:59:42.138+0000 7f56fbbe1200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T02:59:42.802 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:42 smithi148 bash[21615]: debug 2024-03-07T02:59:42.430+0000 7f56fbbe1200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T02:59:42.802 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:42 smithi148 bash[21615]: debug 2024-03-07T02:59:42.546+0000 7f56fbbe1200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T02:59:43.252 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:42 smithi148 bash[21615]: debug 2024-03-07T02:59:42.990+0000 7f56fbbe1200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T02:59:43.252 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 02:59:43 smithi148 bash[21615]: debug 2024-03-07T02:59:43.114+0000 7f56fbbe1200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T02:59:44.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: cluster 2024-03-07T02:59:43.137749+0000 mon.a (mon.0) 71 : cluster 1 Active manager daemon y restarted 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: cluster 2024-03-07T02:59:43.138338+0000 mon.a (mon.0) 72 : cluster 1 Activating manager daemon y 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: cluster 2024-03-07T02:59:43.147465+0000 mon.a (mon.0) 73 : cluster 0 osdmap e3: 0 total, 0 up, 0 in 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: cluster 2024-03-07T02:59:43.147714+0000 mon.a (mon.0) 74 : cluster 0 mgrmap e10: y(active, starting, since 0.00957325s) 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.149335+0000 mon.a (mon.0) 75 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.151376+0000 mon.a (mon.0) 76 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.152412+0000 mon.a (mon.0) 77 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.152681+0000 mon.a (mon.0) 78 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.152934+0000 mon.a (mon.0) 79 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: cluster 2024-03-07T02:59:43.182187+0000 mon.a (mon.0) 80 : cluster 1 Manager daemon y is now available 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.206381+0000 mon.a (mon.0) 81 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.208687+0000 mon.a (mon.0) 82 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.227519+0000 mon.a (mon.0) 83 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T02:59:44.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:44 smithi148 bash[21362]: audit 2024-03-07T02:59:43.232327+0000 mon.a (mon.0) 84 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T02:59:44.757 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout { 2024-03-07T02:59:44.758 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-03-07T02:59:44.758 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-07T02:59:44.758 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout } 2024-03-07T02:59:44.758 INFO:teuthology.orchestra.run.smithi148.stdout:mgr epoch 9 is available 2024-03-07T02:59:44.758 INFO:teuthology.orchestra.run.smithi148.stdout:Generating a dashboard self-signed certificate... 2024-03-07T02:59:45.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:45 smithi148 bash[21362]: cephadm 2024-03-07T02:59:43.489048+0000 mgr.y (mgr.14152) 1 : cephadm 1 [07/Mar/2024:02:59:43] ENGINE Bus STARTING 2024-03-07T02:59:45.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:45 smithi148 bash[21362]: cephadm 2024-03-07T02:59:43.606657+0000 mgr.y (mgr.14152) 2 : cephadm 1 [07/Mar/2024:02:59:43] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T02:59:45.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:45 smithi148 bash[21362]: cephadm 2024-03-07T02:59:43.707923+0000 mgr.y (mgr.14152) 3 : cephadm 1 [07/Mar/2024:02:59:43] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T02:59:45.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:45 smithi148 bash[21362]: cephadm 2024-03-07T02:59:43.708022+0000 mgr.y (mgr.14152) 4 : cephadm 1 [07/Mar/2024:02:59:43] ENGINE Bus STARTED 2024-03-07T02:59:45.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:45 smithi148 bash[21362]: audit 2024-03-07T02:59:44.151213+0000 mgr.y (mgr.14152) 5 : audit 0 from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-07T02:59:45.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:45 smithi148 bash[21362]: cluster 2024-03-07T02:59:44.153117+0000 mon.a (mon.0) 85 : cluster 0 mgrmap e11: y(active, since 1.01497s) 2024-03-07T02:59:46.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:46 smithi148 bash[21362]: audit 2024-03-07T02:59:44.164676+0000 mgr.y (mgr.14152) 6 : audit 0 from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-07T02:59:46.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:46 smithi148 bash[21362]: audit 2024-03-07T02:59:46.025087+0000 mon.a (mon.0) 86 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T02:59:46.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:46 smithi148 bash[21362]: audit 2024-03-07T02:59:46.031090+0000 mon.a (mon.0) 87 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T02:59:46.571 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-03-07T02:59:46.571 INFO:teuthology.orchestra.run.smithi148.stdout:Creating initial admin user... 2024-03-07T02:59:47.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:47 smithi148 bash[21362]: audit 2024-03-07T02:59:45.868847+0000 mgr.y (mgr.14152) 7 : audit 0 from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:47.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:47 smithi148 bash[21362]: cluster 2024-03-07T02:59:46.150453+0000 mon.a (mon.0) 88 : cluster 0 mgrmap e12: y(active, since 3s) 2024-03-07T02:59:48.584 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$U9SAFenp7FB2WRYEy0NJQuHMDs0APx2TSrSYvKJsDyuHTLzTsF1Ii", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1709780388, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-07T02:59:48.584 INFO:teuthology.orchestra.run.smithi148.stdout:Fetching dashboard port number... 2024-03-07T02:59:49.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:49 smithi148 bash[21362]: audit 2024-03-07T02:59:47.786430+0000 mgr.y (mgr.14152) 8 : audit 0 from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T02:59:49.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:49 smithi148 bash[21362]: audit 2024-03-07T02:59:48.026756+0000 mon.a (mon.0) 89 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T02:59:50.187 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stdout 8443 2024-03-07T02:59:50.188 INFO:teuthology.orchestra.run.smithi148.stdout:firewalld does not appear to be present 2024-03-07T02:59:50.188 INFO:teuthology.orchestra.run.smithi148.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-07T02:59:50.190 INFO:teuthology.orchestra.run.smithi148.stdout:Ceph Dashboard is now available at: 2024-03-07T02:59:50.191 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:50.191 INFO:teuthology.orchestra.run.smithi148.stdout: URL: https://smithi148.front.sepia.ceph.com:8443/ 2024-03-07T02:59:50.191 INFO:teuthology.orchestra.run.smithi148.stdout: User: admin 2024-03-07T02:59:50.191 INFO:teuthology.orchestra.run.smithi148.stdout: Password: cx8qctgmrd 2024-03-07T02:59:50.191 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:50.191 INFO:teuthology.orchestra.run.smithi148.stdout:Saving cluster configuration to /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config directory 2024-03-07T02:59:50.192 INFO:teuthology.orchestra.run.smithi148.stdout:Enabling autotune for osd_memory_target 2024-03-07T02:59:50.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:50 smithi148 bash[21362]: audit 2024-03-07T02:59:49.684613+0000 mon.a (mon.0) 90 : audit 0 from='client.? 172.21.15.148:0/2045123214' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-07T02:59:50.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:50 smithi148 bash[21362]: audit 2024-03-07T02:59:49.820603+0000 mon.a (mon.0) 91 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T02:59:51.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:51 smithi148 bash[21362]: cluster 2024-03-07T02:59:50.036759+0000 mon.a (mon.0) 92 : cluster 0 mgrmap e13: y(active, since 6s) 2024-03-07T02:59:51.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:51 smithi148 bash[21362]: audit 2024-03-07T02:59:50.505551+0000 mon.a (mon.0) 93 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout:Or, if you are only running a single cluster on this host: 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.769 INFO:teuthology.orchestra.run.smithi148.stdout:Please consider enabling telemetry to help improve Ceph: 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout: ceph telemetry on 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout:For more information see: 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T02:59:53.770 INFO:teuthology.orchestra.run.smithi148.stdout:Bootstrap complete. 2024-03-07T02:59:53.814 INFO:tasks.cephadm:Fetching config... 2024-03-07T02:59:53.814 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:59:53.814 DEBUG:teuthology.orchestra.run.smithi148:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-07T02:59:53.820 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-07T02:59:53.820 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:59:53.820 DEBUG:teuthology.orchestra.run.smithi148:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-07T02:59:53.867 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-07T02:59:53.867 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:59:53.867 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/keyring of=/dev/stdout 2024-03-07T02:59:53.921 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-07T02:59:53.921 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T02:59:53.921 DEBUG:teuthology.orchestra.run.smithi148:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-07T02:59:53.971 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-07T02:59:53.971 DEBUG:teuthology.orchestra.run.smithi148:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDN6MeDFLa253CKGwQWsXKSzwp32bspl4sUaeZ3WWAs/y3ODnV/eFi2MgQpjhSRyo5kvi+lQvrxkvZjKOqsR/hBVbHzY6TCjANmeaMP7tW2MTclpNBO21oVHTZ6ociGrdfdyaQ4FmFf4DAYGyGRJiHFSisgNTTz3kUjFgVnLsfZ7eIwAngLsLWJHyLdmbe4Zde3oEsm0SQFJVgH8edm3w4fCL13KtNXPDx45Sk73f1FkihEUCQQWnt50EAhYuDNoyTxBN1Ptkpun+5YHhfJAsvtBiHgEh8zyJnvdfhW15giyk/Ua3tosyD9BprStW4rRmL0cmbqNfk4sDGuCVdu9mFzg5GUzHWO22Luqu+AKHTosVKS3YO7QN543LF8VCYEx7Acxy3ylJ5Ef2jaxDBaQcSKw5sPkCQXseOE3x+qtzJ1On1jD8ZPqj4CHpZUVfO9lk0Cn5wsUA30xm9P7CboWBJoOrUvwf4FxfxgOsEYdWJaZzt9Wd5RO9qaAa0hrNBaey0= ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-07T02:59:54.035 INFO:teuthology.orchestra.run.smithi148.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDN6MeDFLa253CKGwQWsXKSzwp32bspl4sUaeZ3WWAs/y3ODnV/eFi2MgQpjhSRyo5kvi+lQvrxkvZjKOqsR/hBVbHzY6TCjANmeaMP7tW2MTclpNBO21oVHTZ6ociGrdfdyaQ4FmFf4DAYGyGRJiHFSisgNTTz3kUjFgVnLsfZ7eIwAngLsLWJHyLdmbe4Zde3oEsm0SQFJVgH8edm3w4fCL13KtNXPDx45Sk73f1FkihEUCQQWnt50EAhYuDNoyTxBN1Ptkpun+5YHhfJAsvtBiHgEh8zyJnvdfhW15giyk/Ua3tosyD9BprStW4rRmL0cmbqNfk4sDGuCVdu9mFzg5GUzHWO22Luqu+AKHTosVKS3YO7QN543LF8VCYEx7Acxy3ylJ5Ef2jaxDBaQcSKw5sPkCQXseOE3x+qtzJ1On1jD8ZPqj4CHpZUVfO9lk0Cn5wsUA30xm9P7CboWBJoOrUvwf4FxfxgOsEYdWJaZzt9Wd5RO9qaAa0hrNBaey0= ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:59:54.045 DEBUG:teuthology.orchestra.run.smithi195:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDN6MeDFLa253CKGwQWsXKSzwp32bspl4sUaeZ3WWAs/y3ODnV/eFi2MgQpjhSRyo5kvi+lQvrxkvZjKOqsR/hBVbHzY6TCjANmeaMP7tW2MTclpNBO21oVHTZ6ociGrdfdyaQ4FmFf4DAYGyGRJiHFSisgNTTz3kUjFgVnLsfZ7eIwAngLsLWJHyLdmbe4Zde3oEsm0SQFJVgH8edm3w4fCL13KtNXPDx45Sk73f1FkihEUCQQWnt50EAhYuDNoyTxBN1Ptkpun+5YHhfJAsvtBiHgEh8zyJnvdfhW15giyk/Ua3tosyD9BprStW4rRmL0cmbqNfk4sDGuCVdu9mFzg5GUzHWO22Luqu+AKHTosVKS3YO7QN543LF8VCYEx7Acxy3ylJ5Ef2jaxDBaQcSKw5sPkCQXseOE3x+qtzJ1On1jD8ZPqj4CHpZUVfO9lk0Cn5wsUA30xm9P7CboWBJoOrUvwf4FxfxgOsEYdWJaZzt9Wd5RO9qaAa0hrNBaey0= ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-07T02:59:54.068 INFO:teuthology.orchestra.run.smithi195.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDN6MeDFLa253CKGwQWsXKSzwp32bspl4sUaeZ3WWAs/y3ODnV/eFi2MgQpjhSRyo5kvi+lQvrxkvZjKOqsR/hBVbHzY6TCjANmeaMP7tW2MTclpNBO21oVHTZ6ociGrdfdyaQ4FmFf4DAYGyGRJiHFSisgNTTz3kUjFgVnLsfZ7eIwAngLsLWJHyLdmbe4Zde3oEsm0SQFJVgH8edm3w4fCL13KtNXPDx45Sk73f1FkihEUCQQWnt50EAhYuDNoyTxBN1Ptkpun+5YHhfJAsvtBiHgEh8zyJnvdfhW15giyk/Ua3tosyD9BprStW4rRmL0cmbqNfk4sDGuCVdu9mFzg5GUzHWO22Luqu+AKHTosVKS3YO7QN543LF8VCYEx7Acxy3ylJ5Ef2jaxDBaQcSKw5sPkCQXseOE3x+qtzJ1On1jD8ZPqj4CHpZUVfO9lk0Cn5wsUA30xm9P7CboWBJoOrUvwf4FxfxgOsEYdWJaZzt9Wd5RO9qaAa0hrNBaey0= ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T02:59:54.078 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-07T02:59:54.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 02:59:54 smithi148 bash[21362]: audit 2024-03-07T02:59:53.277340+0000 mon.a (mon.0) 94 : audit 1 from='client.? 172.21.15.148:0/4208938407' entity='client.admin' 2024-03-07T02:59:58.943 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:00:02.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:01 smithi148 bash[21362]: audit 2024-03-07T03:00:00.745792+0000 mon.a (mon.0) 95 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:02.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:01 smithi148 bash[21362]: audit 2024-03-07T03:00:00.749505+0000 mon.a (mon.0) 96 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:02.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:01 smithi148 bash[21362]: audit 2024-03-07T03:00:00.750534+0000 mon.a (mon.0) 97 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi148", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:02.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:01 smithi148 bash[21362]: audit 2024-03-07T03:00:00.756424+0000 mon.a (mon.0) 98 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:02.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:01 smithi148 bash[21362]: audit 2024-03-07T03:00:00.790747+0000 mon.a (mon.0) 99 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:02.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:01 smithi148 bash[21362]: audit 2024-03-07T03:00:00.797740+0000 mon.a (mon.0) 100 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:02.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:01 smithi148 bash[21362]: audit 2024-03-07T03:00:01.713661+0000 mon.a (mon.0) 101 : audit 1 from='client.? 172.21.15.148:0/4218626223' entity='client.admin' 2024-03-07T03:00:02.264 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-07T03:00:02.264 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-07T03:00:03.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:02 smithi148 bash[21362]: audit 2024-03-07T03:00:01.747478+0000 mon.a (mon.0) 102 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:03.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:02 smithi148 bash[21362]: audit 2024-03-07T03:00:01.751346+0000 mon.a (mon.0) 103 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:04.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:04 smithi148 bash[21362]: cluster 2024-03-07T03:00:03.153916+0000 mgr.y (mgr.14152) 9 : cluster 0 pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:05.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:05 smithi148 bash[21362]: cluster 2024-03-07T03:00:05.154344+0000 mgr.y (mgr.14152) 10 : cluster 0 pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:06.930 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:00:07.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:07 smithi148 bash[21362]: cluster 2024-03-07T03:00:07.154700+0000 mgr.y (mgr.14152) 11 : cluster 0 pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:09.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:09 smithi148 bash[21362]: cluster 2024-03-07T03:00:09.155006+0000 mgr.y (mgr.14152) 12 : cluster 0 pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:09.767 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi195 2024-03-07T03:00:09.767 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T03:00:09.767 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.conf 2024-03-07T03:00:09.773 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T03:00:09.773 DEBUG:teuthology.orchestra.run.smithi195:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:00:09.820 INFO:tasks.cephadm:Adding host smithi195 to orchestrator... 2024-03-07T03:00:09.821 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch host add smithi195 2024-03-07T03:00:10.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.244461+0000 mgr.y (mgr.14152) 13 : audit 0 from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:10.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.249381+0000 mon.a (mon.0) 104 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.299920+0000 mon.a (mon.0) 105 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.301313+0000 mon.a (mon.0) 106 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.301955+0000 mon.a (mon.0) 107 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: cephadm 2024-03-07T03:00:09.302974+0000 mgr.y (mgr.14152) 14 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: cephadm 2024-03-07T03:00:09.417327+0000 mgr.y (mgr.14152) 15 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: cephadm 2024-03-07T03:00:09.582426+0000 mgr.y (mgr.14152) 16 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: cephadm 2024-03-07T03:00:09.747938+0000 mgr.y (mgr.14152) 17 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.916845+0000 mon.a (mon.0) 108 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.920683+0000 mon.a (mon.0) 109 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:10.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:10 smithi148 bash[21362]: audit 2024-03-07T03:00:09.924547+0000 mon.a (mon.0) 110 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:11.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:11 smithi148 bash[21362]: cluster 2024-03-07T03:00:11.155462+0000 mgr.y (mgr.14152) 18 : cluster 0 pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:14.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:14 smithi148 bash[21362]: cluster 2024-03-07T03:00:13.155944+0000 mgr.y (mgr.14152) 19 : cluster 0 pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:14.484 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:00:15.537 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:15 smithi148 bash[21362]: cluster 2024-03-07T03:00:15.156395+0000 mgr.y (mgr.14152) 20 : cluster 0 pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:17.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:17 smithi148 bash[21362]: audit 2024-03-07T03:00:16.905435+0000 mgr.y (mgr.14152) 21 : audit 0 from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi195", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:17.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:17 smithi148 bash[21362]: cluster 2024-03-07T03:00:17.156902+0000 mgr.y (mgr.14152) 22 : cluster 0 pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:19.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:19 smithi148 bash[21362]: cephadm 2024-03-07T03:00:17.674133+0000 mgr.y (mgr.14152) 23 : cephadm 1 Deploying cephadm binary to smithi195 2024-03-07T03:00:20.282 INFO:teuthology.orchestra.run.smithi148.stdout:Added host 'smithi195' with addr '172.21.15.195' 2024-03-07T03:00:20.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:20 smithi148 bash[21362]: cluster 2024-03-07T03:00:19.157296+0000 mgr.y (mgr.14152) 24 : cluster 0 pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:20.875 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch host ls --format=json 2024-03-07T03:00:21.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:21 smithi148 bash[21362]: audit 2024-03-07T03:00:20.282021+0000 mon.a (mon.0) 111 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:21.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:21 smithi148 bash[21362]: cephadm 2024-03-07T03:00:20.282845+0000 mgr.y (mgr.14152) 25 : cephadm 1 Added host smithi195 2024-03-07T03:00:21.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:21 smithi148 bash[21362]: audit 2024-03-07T03:00:20.327606+0000 mon.a (mon.0) 112 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:21.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:21 smithi148 bash[21362]: audit 2024-03-07T03:00:20.650204+0000 mon.a (mon.0) 113 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:21.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:21 smithi148 bash[21362]: cluster 2024-03-07T03:00:21.157690+0000 mgr.y (mgr.14152) 26 : cluster 0 pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:23.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:23 smithi148 bash[21362]: audit 2024-03-07T03:00:22.026071+0000 mon.a (mon.0) 114 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:24.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:24 smithi148 bash[21362]: cluster 2024-03-07T03:00:23.158109+0000 mgr.y (mgr.14152) 27 : cluster 0 pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:24.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:24 smithi148 bash[21362]: audit 2024-03-07T03:00:23.175677+0000 mon.a (mon.0) 115 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:25.542 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:00:25.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:25 smithi148 bash[21362]: cluster 2024-03-07T03:00:25.158517+0000 mgr.y (mgr.14152) 28 : cluster 0 pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:27.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:27 smithi148 bash[21362]: cluster 2024-03-07T03:00:27.158935+0000 mgr.y (mgr.14152) 29 : cluster 0 pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:27.817 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:00:27.817 INFO:teuthology.orchestra.run.smithi148.stdout:[{"addr": "172.21.15.148", "hostname": "smithi148", "labels": [], "status": ""}, {"addr": "172.21.15.195", "hostname": "smithi195", "labels": [], "status": ""}] 2024-03-07T03:00:28.446 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-07T03:00:28.446 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd crush tunables default 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:27.817706+0000 mgr.y (mgr.14152) 30 : audit 0 from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:28.462487+0000 mon.a (mon.0) 116 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:28.465971+0000 mon.a (mon.0) 117 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:28.472632+0000 mon.a (mon.0) 118 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:28.475875+0000 mon.a (mon.0) 119 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:28.476940+0000 mon.a (mon.0) 120 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi195", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:28.478301+0000 mon.a (mon.0) 121 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:29 smithi148 bash[21362]: audit 2024-03-07T03:00:28.479493+0000 mon.a (mon.0) 122 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: cephadm 2024-03-07T03:00:28.481412+0000 mgr.y (mgr.14152) 31 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: cephadm 2024-03-07T03:00:28.646951+0000 mgr.y (mgr.14152) 32 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: cephadm 2024-03-07T03:00:28.811649+0000 mgr.y (mgr.14152) 33 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: cephadm 2024-03-07T03:00:29.026874+0000 mgr.y (mgr.14152) 34 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: cluster 2024-03-07T03:00:29.159375+0000 mgr.y (mgr.14152) 35 : cluster 0 pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: audit 2024-03-07T03:00:29.247975+0000 mon.a (mon.0) 123 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: audit 2024-03-07T03:00:29.253732+0000 mon.a (mon.0) 124 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:30 smithi148 bash[21362]: audit 2024-03-07T03:00:29.259793+0000 mon.a (mon.0) 125 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:31.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:31 smithi148 bash[21362]: cluster 2024-03-07T03:00:31.159871+0000 mgr.y (mgr.14152) 36 : cluster 0 pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:33.122 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:00:34.184 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:34 smithi148 bash[21362]: cluster 2024-03-07T03:00:33.160432+0000 mgr.y (mgr.14152) 37 : cluster 0 pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:36.227 INFO:teuthology.orchestra.run.smithi148.stderr:adjusted tunables profile to default 2024-03-07T03:00:36.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:36 smithi148 bash[21362]: cluster 2024-03-07T03:00:35.160957+0000 mgr.y (mgr.14152) 38 : cluster 0 pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:36.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:36 smithi148 bash[21362]: audit 2024-03-07T03:00:35.580877+0000 mon.a (mon.0) 126 : audit 1 from='client.? 172.21.15.148:0/1020967358' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-07T03:00:36.851 INFO:tasks.cephadm:Adding mon.a on smithi148 2024-03-07T03:00:36.851 INFO:tasks.cephadm:Adding mon.c on smithi148 2024-03-07T03:00:36.851 INFO:tasks.cephadm:Adding mon.b on smithi195 2024-03-07T03:00:36.851 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply mon '3;smithi148:172.21.15.148=a;smithi148:[v2:172.21.15.148:3301,v1:172.21.15.148:6790]=c;smithi195:172.21.15.195=b' 2024-03-07T03:00:37.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:37 smithi148 bash[21362]: audit 2024-03-07T03:00:36.224958+0000 mon.a (mon.0) 127 : audit 1 from='client.? 172.21.15.148:0/1020967358' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-07T03:00:37.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:37 smithi148 bash[21362]: cluster 2024-03-07T03:00:36.225067+0000 mon.a (mon.0) 128 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:00:38.016 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:00:39.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:39 smithi148 bash[21362]: cluster 2024-03-07T03:00:37.161391+0000 mgr.y (mgr.14152) 39 : cluster 0 pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:40.423 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mon update... 2024-03-07T03:00:41.177 DEBUG:teuthology.orchestra.run.smithi148:mon.c> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.c.service 2024-03-07T03:00:41.180 DEBUG:teuthology.orchestra.run.smithi195:mon.b> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.b.service 2024-03-07T03:00:41.182 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-07T03:00:41.182 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph mon dump -f json 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: cluster 2024-03-07T03:00:39.161802+0000 mgr.y (mgr.14152) 40 : cluster 0 pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: audit 2024-03-07T03:00:40.423461+0000 mon.a (mon.0) 129 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: audit 2024-03-07T03:00:40.469921+0000 mon.a (mon.0) 130 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: audit 2024-03-07T03:00:40.473687+0000 mon.a (mon.0) 131 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: audit 2024-03-07T03:00:40.475413+0000 mon.a (mon.0) 132 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: audit 2024-03-07T03:00:40.483500+0000 mon.a (mon.0) 133 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: audit 2024-03-07T03:00:40.487473+0000 mon.a (mon.0) 134 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:00:41.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:41 smithi148 bash[21362]: audit 2024-03-07T03:00:40.489154+0000 mon.a (mon.0) 135 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:42.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:42 smithi148 bash[21362]: audit 2024-03-07T03:00:40.411992+0000 mgr.y (mgr.14152) 41 : audit 0 from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi148:172.21.15.148=a;smithi148:[v2:172.21.15.148:3301,v1:172.21.15.148:6790]=c;smithi195:172.21.15.195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:42.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:42 smithi148 bash[21362]: cephadm 2024-03-07T03:00:40.418054+0000 mgr.y (mgr.14152) 42 : cephadm 1 Saving service mon spec with placement smithi148:172.21.15.148=a;smithi148:[v2:172.21.15.148:3301,v1:172.21.15.148:6790]=c;smithi195:172.21.15.195=b;count:3 2024-03-07T03:00:42.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:42 smithi148 bash[21362]: cephadm 2024-03-07T03:00:40.491071+0000 mgr.y (mgr.14152) 43 : cephadm 1 Deploying daemon mon.b on smithi195 2024-03-07T03:00:42.778 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:00:43.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:43 smithi148 bash[21362]: cluster 2024-03-07T03:00:41.162211+0000 mgr.y (mgr.14152) 44 : cluster 0 pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:45.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:45 smithi148 bash[21362]: cluster 2024-03-07T03:00:43.162628+0000 mgr.y (mgr.14152) 45 : cluster 0 pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:45.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:45 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:46.024 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-03-07T03:00:46.024 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":1,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","modified":"2024-03-07T02:58:38.930467Z","created":"2024-03-07T02:58:38.930467Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:3300","nonce":0},{"type":"v1","addr":"172.21.15.148:6789","nonce":0}]},"addr":"172.21.15.148:6789/0","public_addr":"172.21.15.148:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-07T03:00:46.027 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 1 2024-03-07T03:00:46.041 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:45 smithi195 systemd[1]: Started Ceph mon.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:00:47.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: cluster 2024-03-07T03:00:45.163109+0000 mgr.y (mgr.14152) 46 : cluster 0 pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: audit 2024-03-07T03:00:45.639898+0000 mon.a (mon.0) 136 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: audit 2024-03-07T03:00:45.646045+0000 mon.a (mon.0) 137 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: audit 2024-03-07T03:00:45.674979+0000 mon.a (mon.0) 138 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: audit 2024-03-07T03:00:45.676168+0000 mon.a (mon.0) 139 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:00:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: audit 2024-03-07T03:00:45.677761+0000 mon.a (mon.0) 140 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: cephadm 2024-03-07T03:00:45.682232+0000 mgr.y (mgr.14152) 47 : cephadm 1 Deploying daemon mon.c on smithi148 2024-03-07T03:00:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:46 smithi148 bash[21362]: audit 2024-03-07T03:00:46.025363+0000 mon.a (mon.0) 141 : audit 0 from='client.? 172.21.15.195:0/895123892' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-07T03:00:47.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.287+0000 7fca58b2fc80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T03:00:47.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.287+0000 7fca58b2fc80 0 ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable), process ceph-mon, pid 7 2024-03-07T03:00:47.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.287+0000 7fca58b2fc80 0 pidfile_write: ignore empty --pid-file 2024-03-07T03:00:47.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.291+0000 7fca58b2fc80 0 load: jerasure load: lrc 2024-03-07T03:00:47.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T03:00:47.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Git sha 0 2024-03-07T03:00:47.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Compile date 2024-03-06 23:19:49 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: DB SUMMARY 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: DB Session ID: 4IKX2ZAVTIE6G0PEN3G9 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T03:00:47.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.env: 0x555eac810720 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.info_log: 0x555eafa2b500 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.statistics: (nil) 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.use_fsync: 0 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T03:00:47.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.db_log_dir: 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.wal_dir: 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.write_buffer_manager: 0x555eaecb8320 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T03:00:47.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.unordered_write: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.row_cache: None 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.wal_filter: None 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T03:00:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.wal_compression: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T03:00:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T03:00:47.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_open_files: -1 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Compression algorithms supported: 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kZSTD supported: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T03:00:47.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.merge_operator: 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_filter: None 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555eafa2b600) 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cache_index_and_filter_blocks: 1 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T03:00:47.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: pin_top_level_index_and_filter: 1 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: index_type: 0 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: data_block_index_type: 0 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: index_shortening: 1 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: checksum: 4 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: no_block_cache: 0 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_cache: 0x555eaed3b1f0 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_cache_name: BinnedLRUCache 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_cache_options: 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: capacity : 536870912 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: num_shard_bits : 4 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: strict_capacity_limit : 0 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: high_pri_pool_ratio: 0.000 2024-03-07T03:00:47.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_cache_compressed: (nil) 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: persistent_cache: (nil) 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_size: 4096 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_size_deviation: 10 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_restart_interval: 16 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: index_block_restart_interval: 1 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: metadata_block_size: 4096 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: partition_filters: 0 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: use_delta_encoding: 1 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: filter_policy: bloomfilter 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: whole_key_filtering: 1 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: verify_compression: 0 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: read_amp_bytes_per_bit: 0 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: format_version: 5 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: enable_index_compression: 1 2024-03-07T03:00:47.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: block_align: 0 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: max_auto_readahead_size: 262144 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: prepopulate_block_cache: 0 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: initial_auto_readahead_size: 8192 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: num_file_reads_for_auto_readahead: 2 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression: NoCompression 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.num_levels: 7 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T03:00:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:00:47.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T03:00:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T03:00:47.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.ttl: 2592000 2024-03-07T03:00:47.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.295+0000 7fca58b2fc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-03-07T03:00:47.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dc1c6b4e-2c62-436c-bc71-3d61b7c6d2af 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780447300779, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780447301266, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1622, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709780447, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dc1c6b4e-2c62-436c-bc71-3d61b7c6d2af", "db_session_id": "4IKX2ZAVTIE6G0PEN3G9", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709780447301377, "job": 1, "event": "recovery_finished"} 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555eaeddc000 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 4 rocksdb: DB pointer 0x555eaedca000 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca47329700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca47329700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: ** DB Stats ** 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:00:47.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: ** Compaction Stats [default] ** 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: L0 1/0 1.58 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Sum 1/0 1.58 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: ** Compaction Stats [default] ** 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-03-07T03:00:47.327 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Flush(GB): cumulative 0.000, interval 0.000 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Cumulative compaction: 0.00 GB write, 0.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Interval compaction: 0.00 GB write, 0.28 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Block cache BinnedLRUCache@0x555eaed3b1f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 0 mon.b does not exist in monmap, will attempt to join an existing cluster 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 0 using public_addr v2:172.21.15.195:0/0 -> [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.299+0000 7fca58b2fc80 0 starting mon.b rank -1 at public addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] at bind addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:00:47.328 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.303+0000 7fca58b2fc80 1 mon.b@-1(???) e0 preinit fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 0 mon.b@-1(synchronizing).mds e1 new map 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 0 mon.b@-1(synchronizing).mds e1 print_map 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: e1 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: legacy client fscid: -1 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: No filesystems configured 2024-03-07T03:00:47.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 1 mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 1 mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 1 mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 1 mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 1 mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 1 mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 0 mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:15.156395+0000 mgr.y (mgr.14152) 20 : cluster 0 pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:16.905435+0000 mgr.y (mgr.14152) 21 : audit 0 from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi195", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:17.156902+0000 mgr.y (mgr.14152) 22 : cluster 0 pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:17.674133+0000 mgr.y (mgr.14152) 23 : cephadm 1 Deploying cephadm binary to smithi195 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:19.157296+0000 mgr.y (mgr.14152) 24 : cluster 0 pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:20.282021+0000 mon.a (mon.0) 111 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:20.282845+0000 mgr.y (mgr.14152) 25 : cephadm 1 Added host smithi195 2024-03-07T03:00:47.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:20.327606+0000 mon.a (mon.0) 112 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:47.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:20.650204+0000 mon.a (mon.0) 113 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:21.157690+0000 mgr.y (mgr.14152) 26 : cluster 0 pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:22.026071+0000 mon.a (mon.0) 114 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:23.158109+0000 mgr.y (mgr.14152) 27 : cluster 0 pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:23.175677+0000 mon.a (mon.0) 115 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:25.158517+0000 mgr.y (mgr.14152) 28 : cluster 0 pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:27.158935+0000 mgr.y (mgr.14152) 29 : cluster 0 pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:27.817706+0000 mgr.y (mgr.14152) 30 : audit 0 from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:28.462487+0000 mon.a (mon.0) 116 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:28.465971+0000 mon.a (mon.0) 117 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:28.472632+0000 mon.a (mon.0) 118 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:28.475875+0000 mon.a (mon.0) 119 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:28.476940+0000 mon.a (mon.0) 120 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi195", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:28.478301+0000 mon.a (mon.0) 121 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:28.479493+0000 mon.a (mon.0) 122 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:28.481412+0000 mgr.y (mgr.14152) 31 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:28.646951+0000 mgr.y (mgr.14152) 32 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:28.811649+0000 mgr.y (mgr.14152) 33 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:00:47.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:29.026874+0000 mgr.y (mgr.14152) 34 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:29.159375+0000 mgr.y (mgr.14152) 35 : cluster 0 pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:29.247975+0000 mon.a (mon.0) 123 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:29.253732+0000 mon.a (mon.0) 124 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:29.259793+0000 mon.a (mon.0) 125 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:31.159871+0000 mgr.y (mgr.14152) 36 : cluster 0 pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:33.160432+0000 mgr.y (mgr.14152) 37 : cluster 0 pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:35.160957+0000 mgr.y (mgr.14152) 38 : cluster 0 pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:35.580877+0000 mon.a (mon.0) 126 : audit 1 from='client.? 172.21.15.148:0/1020967358' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:36.224958+0000 mon.a (mon.0) 127 : audit 1 from='client.? 172.21.15.148:0/1020967358' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:36.225067+0000 mon.a (mon.0) 128 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:37.161391+0000 mgr.y (mgr.14152) 39 : cluster 0 pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:39.161802+0000 mgr.y (mgr.14152) 40 : cluster 0 pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.423461+0000 mon.a (mon.0) 129 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.469921+0000 mon.a (mon.0) 130 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:47.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.473687+0000 mon.a (mon.0) 131 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.475413+0000 mon.a (mon.0) 132 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.483500+0000 mon.a (mon.0) 133 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.487473+0000 mon.a (mon.0) 134 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.489154+0000 mon.a (mon.0) 135 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:40.411992+0000 mgr.y (mgr.14152) 41 : audit 0 from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi148:172.21.15.148=a;smithi148:[v2:172.21.15.148:3301,v1:172.21.15.148:6790]=c;smithi195:172.21.15.195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:40.418054+0000 mgr.y (mgr.14152) 42 : cephadm 1 Saving service mon spec with placement smithi148:172.21.15.148=a;smithi148:[v2:172.21.15.148:3301,v1:172.21.15.148:6790]=c;smithi195:172.21.15.195=b;count:3 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:40.491071+0000 mgr.y (mgr.14152) 43 : cephadm 1 Deploying daemon mon.b on smithi195 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:41.162211+0000 mgr.y (mgr.14152) 44 : cluster 0 pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:43.162628+0000 mgr.y (mgr.14152) 45 : cluster 0 pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cluster 2024-03-07T03:00:45.163109+0000 mgr.y (mgr.14152) 46 : cluster 0 pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:45.639898+0000 mon.a (mon.0) 136 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:45.646045+0000 mon.a (mon.0) 137 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:45.674979+0000 mon.a (mon.0) 138 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:45.676168+0000 mon.a (mon.0) 139 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:45.677761+0000 mon.a (mon.0) 140 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:00:47.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: cephadm 2024-03-07T03:00:45.682232+0000 mgr.y (mgr.14152) 47 : cephadm 1 Deploying daemon mon.c on smithi148 2024-03-07T03:00:47.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: audit 2024-03-07T03:00:46.025363+0000 mon.a (mon.0) 141 : audit 0 from='client.? 172.21.15.195:0/895123892' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-07T03:00:47.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:47 smithi195 bash[23221]: debug 2024-03-07T03:00:47.323+0000 7fca4a32f700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-07T03:00:48.058 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-07T03:00:48.058 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph mon dump -f json 2024-03-07T03:00:49.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:00:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:49.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:49.301 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:00:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:49.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:00:50.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:50 smithi148 bash[31149]: debug 2024-03-07T03:00:50.169+0000 7f83e1652700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-03-07T03:00:51.714 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:00:52.752 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:47.163534+0000 mgr.y (mgr.14152) 48 : cluster 0 pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:52.752 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:47.333544+0000 mon.a (mon.0) 142 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.752 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:47.338357+0000 mon.a (mon.0) 143 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:00:52.752 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:47.338730+0000 mon.a (mon.0) 144 : cluster 1 mon.a calling monitor election 2024-03-07T03:00:52.752 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:47.340596+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:48.333135+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:49.163915+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:49.332510+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:49.333211+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:50.175994+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:50.333472+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:51.164329+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:51.176846+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:51.333328+0000 mon.a (mon.0) 151 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:52.176993+0000 mon.a (mon.0) 152 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:52.333527+0000 mon.a (mon.0) 153 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:52.363579+0000 mon.a (mon.0) 154 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:52.369063+0000 mon.a (mon.0) 155 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:52.389069+0000 mon.a (mon.0) 156 : cluster 0 fsmap 2024-03-07T03:00:52.753 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:52.389124+0000 mon.a (mon.0) 157 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:00:52.754 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:52.389398+0000 mon.a (mon.0) 158 : cluster 0 mgrmap e13: y(active, since 69s) 2024-03-07T03:00:52.754 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: cluster 2024-03-07T03:00:52.396089+0000 mon.a (mon.0) 159 : cluster 1 overall HEALTH_OK 2024-03-07T03:00:52.754 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:52.400665+0000 mon.a (mon.0) 160 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:52.754 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:52.408349+0000 mon.a (mon.0) 161 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:52.754 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:52.417616+0000 mon.a (mon.0) 162 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:52.754 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:52 smithi195 bash[23221]: audit 2024-03-07T03:00:52.424808+0000 mon.a (mon.0) 163 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:47.163534+0000 mgr.y (mgr.14152) 48 : cluster 0 pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:47.333544+0000 mon.a (mon.0) 142 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:47.338357+0000 mon.a (mon.0) 143 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:47.338730+0000 mon.a (mon.0) 144 : cluster 1 mon.a calling monitor election 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:47.340596+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:48.333135+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:49.163915+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:49.332510+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:49.333211+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:50.175994+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:52.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:50.333472+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:51.164329+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:51.176846+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:51.333328+0000 mon.a (mon.0) 151 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:52.176993+0000 mon.a (mon.0) 152 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:52.333527+0000 mon.a (mon.0) 153 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:52.363579+0000 mon.a (mon.0) 154 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:52.369063+0000 mon.a (mon.0) 155 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:52.389069+0000 mon.a (mon.0) 156 : cluster 0 fsmap 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:52.389124+0000 mon.a (mon.0) 157 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:52.389398+0000 mon.a (mon.0) 158 : cluster 0 mgrmap e13: y(active, since 69s) 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: cluster 2024-03-07T03:00:52.396089+0000 mon.a (mon.0) 159 : cluster 1 overall HEALTH_OK 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:52.400665+0000 mon.a (mon.0) 160 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:52.408349+0000 mon.a (mon.0) 161 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:52.417616+0000 mon.a (mon.0) 162 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:52.805 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:52 smithi148 bash[21362]: audit 2024-03-07T03:00:52.424808+0000 mon.a (mon.0) 163 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:53.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:53 smithi148 bash[21362]: audit 2024-03-07T03:00:52.491051+0000 mon.a (mon.0) 164 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:53.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:53 smithi148 bash[21362]: audit 2024-03-07T03:00:53.177194+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:53.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:53 smithi148 bash[21362]: audit 2024-03-07T03:00:53.230615+0000 mon.a (mon.0) 166 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:53.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:53 smithi148 bash[21362]: audit 2024-03-07T03:00:53.333822+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:53.825 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:53 smithi195 bash[23221]: audit 2024-03-07T03:00:52.491051+0000 mon.a (mon.0) 164 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:53.825 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:53 smithi195 bash[23221]: audit 2024-03-07T03:00:53.177194+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:53.825 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:53 smithi195 bash[23221]: audit 2024-03-07T03:00:53.230615+0000 mon.a (mon.0) 166 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:53.825 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:53 smithi195 bash[23221]: audit 2024-03-07T03:00:53.333822+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:47.163534+0000 mgr.y (mgr.14152) 48 : cluster 0 pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:47.333544+0000 mon.a (mon.0) 142 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:47.338357+0000 mon.a (mon.0) 143 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:47.338730+0000 mon.a (mon.0) 144 : cluster 1 mon.a calling monitor election 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:47.340596+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:48.333135+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:49.163915+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:49.332510+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:49.333211+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:50.175994+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:54.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:50.333472+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:51.164329+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:51.176846+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:51.333328+0000 mon.a (mon.0) 151 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:52.176993+0000 mon.a (mon.0) 152 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:52.333527+0000 mon.a (mon.0) 153 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:52.363579+0000 mon.a (mon.0) 154 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:52.369063+0000 mon.a (mon.0) 155 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:52.389069+0000 mon.a (mon.0) 156 : cluster 0 fsmap 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:52.389124+0000 mon.a (mon.0) 157 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:52.389398+0000 mon.a (mon.0) 158 : cluster 0 mgrmap e13: y(active, since 69s) 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: cluster 2024-03-07T03:00:52.396089+0000 mon.a (mon.0) 159 : cluster 1 overall HEALTH_OK 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:52.400665+0000 mon.a (mon.0) 160 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:52.408349+0000 mon.a (mon.0) 161 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:54.554 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:52.417616+0000 mon.a (mon.0) 162 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:54.555 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:52.424808+0000 mon.a (mon.0) 163 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:54.555 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:52.491051+0000 mon.a (mon.0) 164 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:00:54.555 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:53.177194+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:54.555 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:53.230615+0000 mon.a (mon.0) 166 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:54.555 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:00:54 smithi148 bash[31149]: audit 2024-03-07T03:00:53.333822+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:59.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:54.189169+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:54.193174+0000 mon.a (mon.0) 169 : cluster 1 mon.a calling monitor election 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:54.217615+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:54.217989+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:54.218356+0000 mon.a (mon.0) 172 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:55.165196+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:55.177684+0000 mon.a (mon.0) 173 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:56.177744+0000 mon.a (mon.0) 174 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:57.165678+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:57.177989+0000 mon.a (mon.0) 175 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:58.178112+0000 mon.a (mon.0) 176 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:59.178109+0000 mon.a (mon.0) 177 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.217685+0000 mon.a (mon.0) 178 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.223589+0000 mon.a (mon.0) 179 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.241114+0000 mon.a (mon.0) 180 : cluster 0 fsmap 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.241199+0000 mon.a (mon.0) 181 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.241485+0000 mon.a (mon.0) 182 : cluster 0 mgrmap e13: y(active, since 76s) 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.241793+0000 mon.a (mon.0) 183 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.249447+0000 mon.a (mon.0) 184 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.249514+0000 mon.a (mon.0) 185 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.249557+0000 mon.a (mon.0) 186 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0] is down (out of quorum) 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:59.254727+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:59.260714+0000 mon.a (mon.0) 188 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:59.267055+0000 mon.a (mon.0) 189 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:59.554 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:00:59 smithi148 bash[21362]: audit 2024-03-07T03:00:59.273492+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:54.189169+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:54.193174+0000 mon.a (mon.0) 169 : cluster 1 mon.a calling monitor election 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:54.217615+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:54.217989+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:54.218356+0000 mon.a (mon.0) 172 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:55.165196+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:55.177684+0000 mon.a (mon.0) 173 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:56.177744+0000 mon.a (mon.0) 174 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:57.165678+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:00:59.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:57.177989+0000 mon.a (mon.0) 175 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:58.178112+0000 mon.a (mon.0) 176 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:59.178109+0000 mon.a (mon.0) 177 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.217685+0000 mon.a (mon.0) 178 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.223589+0000 mon.a (mon.0) 179 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.241114+0000 mon.a (mon.0) 180 : cluster 0 fsmap 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.241199+0000 mon.a (mon.0) 181 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.241485+0000 mon.a (mon.0) 182 : cluster 0 mgrmap e13: y(active, since 76s) 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.241793+0000 mon.a (mon.0) 183 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.249447+0000 mon.a (mon.0) 184 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.249514+0000 mon.a (mon.0) 185 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.249557+0000 mon.a (mon.0) 186 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0] is down (out of quorum) 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:59.254727+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:59.260714+0000 mon.a (mon.0) 188 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:59.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:59.267055+0000 mon.a (mon.0) 189 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:00:59.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:00:59 smithi195 bash[23221]: audit 2024-03-07T03:00:59.273492+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: cluster 2024-03-07T03:00:59.166167+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:00:59.743752+0000 mon.a (mon.0) 191 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:00:59.745609+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: cephadm 2024-03-07T03:00:59.748080+0000 mgr.y (mgr.14152) 55 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: cephadm 2024-03-07T03:00:59.748490+0000 mgr.y (mgr.14152) 56 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: cephadm 2024-03-07T03:00:59.913377+0000 mgr.y (mgr.14152) 57 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: cephadm 2024-03-07T03:00:59.914581+0000 mgr.y (mgr.14152) 58 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.084954+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.090333+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.096248+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.101390+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.107820+0000 mon.a (mon.0) 197 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.181102+0000 mon.a (mon.0) 198 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.181440+0000 mon.a (mon.0) 199 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.186801+0000 mon.a (mon.0) 200 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.192650+0000 mon.a (mon.0) 201 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.198261+0000 mon.a (mon.0) 202 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.199455+0000 mon.a (mon.0) 203 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.201041+0000 mon.a (mon.0) 204 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:00.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:00 smithi195 bash[23221]: audit 2024-03-07T03:01:00.202627+0000 mon.a (mon.0) 205 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:00.562 INFO:teuthology.orchestra.run.smithi195.stdout: 2024-03-07T03:01:00.562 INFO:teuthology.orchestra.run.smithi195.stdout:{"epoch":3,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","modified":"2024-03-07T03:00:54.186254Z","created":"2024-03-07T02:58:38.930467Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:3300","nonce":0},{"type":"v1","addr":"172.21.15.148:6789","nonce":0}]},"addr":"172.21.15.148:6789/0","public_addr":"172.21.15.148:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:3300","nonce":0},{"type":"v1","addr":"172.21.15.195:6789","nonce":0}]},"addr":"172.21.15.195:6789/0","public_addr":"172.21.15.195:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:3301","nonce":0},{"type":"v1","addr":"172.21.15.148:6790","nonce":0}]},"addr":"172.21.15.148:6790/0","public_addr":"172.21.15.148:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-07T03:01:00.565 INFO:teuthology.orchestra.run.smithi195.stderr:dumped monmap epoch 3 2024-03-07T03:01:00.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: cluster 2024-03-07T03:00:59.166167+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:00.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:00:59.743752+0000 mon.a (mon.0) 191 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:00.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:00:59.745609+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:00.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: cephadm 2024-03-07T03:00:59.748080+0000 mgr.y (mgr.14152) 55 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:01:00.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: cephadm 2024-03-07T03:00:59.748490+0000 mgr.y (mgr.14152) 56 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:01:00.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: cephadm 2024-03-07T03:00:59.913377+0000 mgr.y (mgr.14152) 57 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: cephadm 2024-03-07T03:00:59.914581+0000 mgr.y (mgr.14152) 58 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.084954+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.090333+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.096248+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.101390+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.107820+0000 mon.a (mon.0) 197 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.181102+0000 mon.a (mon.0) 198 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.181440+0000 mon.a (mon.0) 199 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.186801+0000 mon.a (mon.0) 200 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.192650+0000 mon.a (mon.0) 201 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.198261+0000 mon.a (mon.0) 202 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.199455+0000 mon.a (mon.0) 203 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.201041+0000 mon.a (mon.0) 204 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:00.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:00 smithi148 bash[21362]: audit 2024-03-07T03:01:00.202627+0000 mon.a (mon.0) 205 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:01.251 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-07T03:01:01.252 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph config generate-minimal-conf 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:00:56.188472+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.219701+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.220292+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.221046+0000 mon.a (mon.0) 208 : cluster 1 mon.a calling monitor election 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.244743+0000 mon.a (mon.0) 209 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.252279+0000 mon.a (mon.0) 210 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.253293+0000 mon.a (mon.0) 211 : cluster 0 fsmap 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.253359+0000 mon.a (mon.0) 212 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.253634+0000 mon.a (mon.0) 213 : cluster 0 mgrmap e13: y(active, since 78s) 2024-03-07T03:01:01.470 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.253930+0000 mon.a (mon.0) 214 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:01:01.471 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.253973+0000 mon.a (mon.0) 215 : cluster 1 Cluster is now healthy 2024-03-07T03:01:01.471 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.272084+0000 mon.a (mon.0) 216 : cluster 1 overall HEALTH_OK 2024-03-07T03:01:01.471 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:54.189169+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:01:01.471 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:54.193174+0000 mon.a (mon.0) 169 : cluster 1 mon.a calling monitor election 2024-03-07T03:01:01.471 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:54.217615+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:01:01.471 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:54.217989+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:01:01.471 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:54.218356+0000 mon.a (mon.0) 172 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:55.165196+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:55.177684+0000 mon.a (mon.0) 173 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:56.177744+0000 mon.a (mon.0) 174 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:57.165678+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:57.177989+0000 mon.a (mon.0) 175 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:58.178112+0000 mon.a (mon.0) 176 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:59.178109+0000 mon.a (mon.0) 177 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.217685+0000 mon.a (mon.0) 178 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.223589+0000 mon.a (mon.0) 179 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.241114+0000 mon.a (mon.0) 180 : cluster 0 fsmap 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.241199+0000 mon.a (mon.0) 181 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.241485+0000 mon.a (mon.0) 182 : cluster 0 mgrmap e13: y(active, since 76s) 2024-03-07T03:01:01.472 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.241793+0000 mon.a (mon.0) 183 : cluster 3 Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.249447+0000 mon.a (mon.0) 184 : cluster 3 Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.249514+0000 mon.a (mon.0) 185 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.249557+0000 mon.a (mon.0) 186 : cluster 3 mon.c (rank 2) addr [v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0] is down (out of quorum) 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:59.254727+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:59.260714+0000 mon.a (mon.0) 188 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:59.267055+0000 mon.a (mon.0) 189 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:59.273492+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:59.166167+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:59.743752+0000 mon.a (mon.0) 191 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:00:59.745609+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cephadm 2024-03-07T03:00:59.748080+0000 mgr.y (mgr.14152) 55 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cephadm 2024-03-07T03:00:59.748490+0000 mgr.y (mgr.14152) 56 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cephadm 2024-03-07T03:00:59.913377+0000 mgr.y (mgr.14152) 57 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cephadm 2024-03-07T03:00:59.914581+0000 mgr.y (mgr.14152) 58 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:01:01.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.084954+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.090333+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.096248+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.101390+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.107820+0000 mon.a (mon.0) 197 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.181102+0000 mon.a (mon.0) 198 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.181440+0000 mon.a (mon.0) 199 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.186801+0000 mon.a (mon.0) 200 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.192650+0000 mon.a (mon.0) 201 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.198261+0000 mon.a (mon.0) 202 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.199455+0000 mon.a (mon.0) 203 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.201041+0000 mon.a (mon.0) 204 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: audit 2024-03-07T03:01:00.202627+0000 mon.a (mon.0) 205 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:00:56.188472+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.219701+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2024-03-07T03:01:01.474 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.220292+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.221046+0000 mon.a (mon.0) 208 : cluster 1 mon.a calling monitor election 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.244743+0000 mon.a (mon.0) 209 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.252279+0000 mon.a (mon.0) 210 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.253293+0000 mon.a (mon.0) 211 : cluster 0 fsmap 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.253359+0000 mon.a (mon.0) 212 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.253634+0000 mon.a (mon.0) 213 : cluster 0 mgrmap e13: y(active, since 78s) 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.253930+0000 mon.a (mon.0) 214 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.253973+0000 mon.a (mon.0) 215 : cluster 1 Cluster is now healthy 2024-03-07T03:01:01.475 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.272084+0000 mon.a (mon.0) 216 : cluster 1 overall HEALTH_OK 2024-03-07T03:01:01.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:00:56.188472+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:01:01.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.219701+0000 mon.b (mon.1) 3 : cluster 1 mon.b calling monitor election 2024-03-07T03:01:01.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.220292+0000 mon.c (mon.2) 2 : cluster 1 mon.c calling monitor election 2024-03-07T03:01:01.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.221046+0000 mon.a (mon.0) 208 : cluster 1 mon.a calling monitor election 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.244743+0000 mon.a (mon.0) 209 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.252279+0000 mon.a (mon.0) 210 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.253293+0000 mon.a (mon.0) 211 : cluster 0 fsmap 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.253359+0000 mon.a (mon.0) 212 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.253634+0000 mon.a (mon.0) 213 : cluster 0 mgrmap e13: y(active, since 78s) 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.253930+0000 mon.a (mon.0) 214 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.253973+0000 mon.a (mon.0) 215 : cluster 1 Cluster is now healthy 2024-03-07T03:01:01.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.272084+0000 mon.a (mon.0) 216 : cluster 1 overall HEALTH_OK 2024-03-07T03:01:03.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[21362]: cluster 2024-03-07T03:01:01.166607+0000 mgr.y (mgr.14152) 61 : cluster 0 pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:03.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[21362]: audit 2024-03-07T03:01:02.117867+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:03.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[21362]: audit 2024-03-07T03:01:02.125327+0000 mon.a (mon.0) 218 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:03.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[21362]: audit 2024-03-07T03:01:02.174467+0000 mon.a (mon.0) 219 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:03.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[21362]: audit 2024-03-07T03:01:02.176311+0000 mon.a (mon.0) 220 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:03.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[21362]: audit 2024-03-07T03:01:02.178147+0000 mon.a (mon.0) 221 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:03.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[21362]: audit 2024-03-07T03:01:02.178751+0000 mon.a (mon.0) 222 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:03.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[31149]: cluster 2024-03-07T03:01:01.166607+0000 mgr.y (mgr.14152) 61 : cluster 0 pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:03.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[31149]: audit 2024-03-07T03:01:02.117867+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:03.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[31149]: audit 2024-03-07T03:01:02.125327+0000 mon.a (mon.0) 218 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:03.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[31149]: audit 2024-03-07T03:01:02.174467+0000 mon.a (mon.0) 219 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:03.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[31149]: audit 2024-03-07T03:01:02.176311+0000 mon.a (mon.0) 220 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:03.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[31149]: audit 2024-03-07T03:01:02.178147+0000 mon.a (mon.0) 221 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:03.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:03 smithi148 bash[31149]: audit 2024-03-07T03:01:02.178751+0000 mon.a (mon.0) 222 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:03.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:03 smithi195 bash[23221]: cluster 2024-03-07T03:01:01.166607+0000 mgr.y (mgr.14152) 61 : cluster 0 pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:03.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:03 smithi195 bash[23221]: audit 2024-03-07T03:01:02.117867+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:03.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:03 smithi195 bash[23221]: audit 2024-03-07T03:01:02.125327+0000 mon.a (mon.0) 218 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:03.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:03 smithi195 bash[23221]: audit 2024-03-07T03:01:02.174467+0000 mon.a (mon.0) 219 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:03.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:03 smithi195 bash[23221]: audit 2024-03-07T03:01:02.176311+0000 mon.a (mon.0) 220 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:03.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:03 smithi195 bash[23221]: audit 2024-03-07T03:01:02.178147+0000 mon.a (mon.0) 221 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:03.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:03 smithi195 bash[23221]: audit 2024-03-07T03:01:02.178751+0000 mon.a (mon.0) 222 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:01:04.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[21362]: cephadm 2024-03-07T03:01:02.173811+0000 mgr.y (mgr.14152) 62 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:01:04.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[21362]: cephadm 2024-03-07T03:01:02.180532+0000 mgr.y (mgr.14152) 63 : cephadm 1 Reconfiguring daemon mon.c on smithi148 2024-03-07T03:01:04.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[21362]: audit 2024-03-07T03:01:03.748585+0000 mon.a (mon.0) 223 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:04.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[21362]: audit 2024-03-07T03:01:03.754606+0000 mon.a (mon.0) 224 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:04.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[21362]: audit 2024-03-07T03:01:03.799053+0000 mon.a (mon.0) 225 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:04.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[21362]: audit 2024-03-07T03:01:03.800310+0000 mon.a (mon.0) 226 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:04.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[21362]: audit 2024-03-07T03:01:03.801578+0000 mon.a (mon.0) 227 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:04.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[31149]: cephadm 2024-03-07T03:01:02.173811+0000 mgr.y (mgr.14152) 62 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:01:04.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[31149]: cephadm 2024-03-07T03:01:02.180532+0000 mgr.y (mgr.14152) 63 : cephadm 1 Reconfiguring daemon mon.c on smithi148 2024-03-07T03:01:04.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[31149]: audit 2024-03-07T03:01:03.748585+0000 mon.a (mon.0) 223 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:04.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[31149]: audit 2024-03-07T03:01:03.754606+0000 mon.a (mon.0) 224 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:04.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[31149]: audit 2024-03-07T03:01:03.799053+0000 mon.a (mon.0) 225 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:04.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[31149]: audit 2024-03-07T03:01:03.800310+0000 mon.a (mon.0) 226 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:04.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:04 smithi148 bash[31149]: audit 2024-03-07T03:01:03.801578+0000 mon.a (mon.0) 227 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:04.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:04 smithi195 bash[23221]: cephadm 2024-03-07T03:01:02.173811+0000 mgr.y (mgr.14152) 62 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:01:04.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:04 smithi195 bash[23221]: cephadm 2024-03-07T03:01:02.180532+0000 mgr.y (mgr.14152) 63 : cephadm 1 Reconfiguring daemon mon.c on smithi148 2024-03-07T03:01:04.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:04 smithi195 bash[23221]: audit 2024-03-07T03:01:03.748585+0000 mon.a (mon.0) 223 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:04.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:04 smithi195 bash[23221]: audit 2024-03-07T03:01:03.754606+0000 mon.a (mon.0) 224 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:04.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:04 smithi195 bash[23221]: audit 2024-03-07T03:01:03.799053+0000 mon.a (mon.0) 225 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:01:04.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:04 smithi195 bash[23221]: audit 2024-03-07T03:01:03.800310+0000 mon.a (mon.0) 226 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:01:04.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:04 smithi195 bash[23221]: audit 2024-03-07T03:01:03.801578+0000 mon.a (mon.0) 227 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:05.189 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:05 smithi195 bash[23221]: cluster 2024-03-07T03:01:03.166947+0000 mgr.y (mgr.14152) 64 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:05.189 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:05 smithi195 bash[23221]: cephadm 2024-03-07T03:01:03.798536+0000 mgr.y (mgr.14152) 65 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:01:05.189 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:05 smithi195 bash[23221]: cephadm 2024-03-07T03:01:03.803458+0000 mgr.y (mgr.14152) 66 : cephadm 1 Reconfiguring daemon mon.b on smithi195 2024-03-07T03:01:05.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:05 smithi148 bash[21362]: cluster 2024-03-07T03:01:03.166947+0000 mgr.y (mgr.14152) 64 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:05.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:05 smithi148 bash[21362]: cephadm 2024-03-07T03:01:03.798536+0000 mgr.y (mgr.14152) 65 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:01:05.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:05 smithi148 bash[21362]: cephadm 2024-03-07T03:01:03.803458+0000 mgr.y (mgr.14152) 66 : cephadm 1 Reconfiguring daemon mon.b on smithi195 2024-03-07T03:01:05.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:05 smithi148 bash[31149]: cluster 2024-03-07T03:01:03.166947+0000 mgr.y (mgr.14152) 64 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:05.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:05 smithi148 bash[31149]: cephadm 2024-03-07T03:01:03.798536+0000 mgr.y (mgr.14152) 65 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:01:05.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:05 smithi148 bash[31149]: cephadm 2024-03-07T03:01:03.803458+0000 mgr.y (mgr.14152) 66 : cephadm 1 Reconfiguring daemon mon.b on smithi195 2024-03-07T03:01:05.934 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:01:06.955 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[21362]: cluster 2024-03-07T03:01:05.167348+0000 mgr.y (mgr.14152) 67 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:06.955 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[21362]: audit 2024-03-07T03:01:05.638465+0000 mon.a (mon.0) 228 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:06.955 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[21362]: audit 2024-03-07T03:01:05.646083+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:06.955 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[21362]: audit 2024-03-07T03:01:05.673413+0000 mon.a (mon.0) 230 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:06.955 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[21362]: audit 2024-03-07T03:01:05.677300+0000 mon.a (mon.0) 231 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:06.955 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[21362]: audit 2024-03-07T03:01:05.679208+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[21362]: audit 2024-03-07T03:01:05.689124+0000 mon.a (mon.0) 233 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[31149]: cluster 2024-03-07T03:01:05.167348+0000 mgr.y (mgr.14152) 67 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[31149]: audit 2024-03-07T03:01:05.638465+0000 mon.a (mon.0) 228 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[31149]: audit 2024-03-07T03:01:05.646083+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[31149]: audit 2024-03-07T03:01:05.673413+0000 mon.a (mon.0) 230 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[31149]: audit 2024-03-07T03:01:05.677300+0000 mon.a (mon.0) 231 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[31149]: audit 2024-03-07T03:01:05.679208+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:06.956 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:06 smithi148 bash[31149]: audit 2024-03-07T03:01:05.689124+0000 mon.a (mon.0) 233 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:07.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:06 smithi195 bash[23221]: cluster 2024-03-07T03:01:05.167348+0000 mgr.y (mgr.14152) 67 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:07.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:06 smithi195 bash[23221]: audit 2024-03-07T03:01:05.638465+0000 mon.a (mon.0) 228 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:07.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:06 smithi195 bash[23221]: audit 2024-03-07T03:01:05.646083+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:07.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:06 smithi195 bash[23221]: audit 2024-03-07T03:01:05.673413+0000 mon.a (mon.0) 230 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:07.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:06 smithi195 bash[23221]: audit 2024-03-07T03:01:05.677300+0000 mon.a (mon.0) 231 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:07.076 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:06 smithi195 bash[23221]: audit 2024-03-07T03:01:05.679208+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:07.076 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:06 smithi195 bash[23221]: audit 2024-03-07T03:01:05.689124+0000 mon.a (mon.0) 233 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:08.361 INFO:teuthology.orchestra.run.smithi148.stdout:# minimal ceph.conf for b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:01:08.361 INFO:teuthology.orchestra.run.smithi148.stdout:[global] 2024-03-07T03:01:08.361 INFO:teuthology.orchestra.run.smithi148.stdout: fsid = b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:01:08.361 INFO:teuthology.orchestra.run.smithi148.stdout: mon_host = [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] [v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0] 2024-03-07T03:01:08.933 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-07T03:01:08.933 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T03:01:08.933 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/ceph/ceph.conf 2024-03-07T03:01:08.947 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T03:01:08.947 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:01:09.006 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T03:01:09.006 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.conf 2024-03-07T03:01:09.020 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T03:01:09.020 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:01:09.080 INFO:tasks.cephadm:Adding mgr.y on smithi148 2024-03-07T03:01:09.080 INFO:tasks.cephadm:Adding mgr.x on smithi195 2024-03-07T03:01:09.080 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply mgr '2;smithi148=y;smithi195=x' 2024-03-07T03:01:09.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:09 smithi148 bash[21362]: cluster 2024-03-07T03:01:07.167770+0000 mgr.y (mgr.14152) 68 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:09.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:09 smithi148 bash[21362]: audit 2024-03-07T03:01:08.362711+0000 mon.a (mon.0) 234 : audit 0 from='client.? 172.21.15.148:0/2348420542' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:09.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:09 smithi148 bash[31149]: cluster 2024-03-07T03:01:07.167770+0000 mgr.y (mgr.14152) 68 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:09.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:09 smithi148 bash[31149]: audit 2024-03-07T03:01:08.362711+0000 mon.a (mon.0) 234 : audit 0 from='client.? 172.21.15.148:0/2348420542' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:09.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:09 smithi195 bash[23221]: cluster 2024-03-07T03:01:07.167770+0000 mgr.y (mgr.14152) 68 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:09.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:09 smithi195 bash[23221]: audit 2024-03-07T03:01:08.362711+0000 mon.a (mon.0) 234 : audit 0 from='client.? 172.21.15.148:0/2348420542' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:11.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:11 smithi148 bash[21362]: cluster 2024-03-07T03:01:09.168219+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:11.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:11 smithi148 bash[31149]: cluster 2024-03-07T03:01:09.168219+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:11.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:11 smithi195 bash[23221]: cluster 2024-03-07T03:01:09.168219+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:12.776 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:01:13.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:13 smithi195 bash[23221]: cluster 2024-03-07T03:01:11.168617+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:13.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:13 smithi148 bash[21362]: cluster 2024-03-07T03:01:11.168617+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:13.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:13 smithi148 bash[31149]: cluster 2024-03-07T03:01:11.168617+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:15 smithi195 bash[23221]: cluster 2024-03-07T03:01:13.169027+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:15.328 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled mgr update... 2024-03-07T03:01:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:15 smithi148 bash[21362]: cluster 2024-03-07T03:01:13.169027+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:15 smithi148 bash[31149]: cluster 2024-03-07T03:01:13.169027+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:16.408 DEBUG:teuthology.orchestra.run.smithi195:mgr.x> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.x.service 2024-03-07T03:01:16.410 INFO:tasks.cephadm:Deploying OSDs... 2024-03-07T03:01:16.410 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T03:01:16.411 DEBUG:teuthology.orchestra.run.smithi148:> dd if=/scratch_devs of=/dev/stdout 2024-03-07T03:01:16.417 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-07T03:01:16.417 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_1 2024-03-07T03:01:16.466 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-07T03:01:16.466 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:16.466 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 887 Links: 1 2024-03-07T03:01:16.467 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:16.467 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-03-07 02:46:37.417386343 +0000 2024-03-07T03:01:16.467 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-03-07 02:46:37.153388698 +0000 2024-03-07T03:01:16.467 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-03-07 02:46:37.153388698 +0000 2024-03-07T03:01:16.467 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: - 2024-03-07T03:01:16.467 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-07T03:01:16.521 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-03-07T03:01:16.521 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-03-07T03:01:16.521 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.000412241 s, 1.2 MB/s 2024-03-07T03:01:16.522 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-07T03:01:16.575 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_2 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 898 Links: 1 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-03-07 02:46:37.937381703 +0000 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-03-07 02:46:37.673384058 +0000 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-03-07 02:46:37.673384058 +0000 2024-03-07T03:01:16.622 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: - 2024-03-07T03:01:16.622 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: cluster 2024-03-07T03:01:15.169427+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.318751+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi148=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: cephadm 2024-03-07T03:01:15.322331+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi148=y;smithi195=x;count:2 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.328947+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.347607+0000 mon.a (mon.0) 236 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.351351+0000 mon.a (mon.0) 237 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.353139+0000 mon.a (mon.0) 238 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.362389+0000 mon.a (mon.0) 239 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.365476+0000 mon.a (mon.0) 240 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.371845+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-07T03:01:16.675 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.373588+0000 mon.a (mon.0) 242 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: audit 2024-03-07T03:01:15.375463+0000 mon.a (mon.0) 243 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[21362]: cephadm 2024-03-07T03:01:15.377487+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: cluster 2024-03-07T03:01:15.169427+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.318751+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi148=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: cephadm 2024-03-07T03:01:15.322331+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi148=y;smithi195=x;count:2 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.328947+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.347607+0000 mon.a (mon.0) 236 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.351351+0000 mon.a (mon.0) 237 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.353139+0000 mon.a (mon.0) 238 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.362389+0000 mon.a (mon.0) 239 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.365476+0000 mon.a (mon.0) 240 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.371845+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-07T03:01:16.676 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.373588+0000 mon.a (mon.0) 242 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:01:16.677 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: audit 2024-03-07T03:01:15.375463+0000 mon.a (mon.0) 243 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:16.677 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:16 smithi148 bash[31149]: cephadm 2024-03-07T03:01:15.377487+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:01:16.677 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-03-07T03:01:16.677 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-03-07T03:01:16.677 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.000438426 s, 1.2 MB/s 2024-03-07T03:01:16.678 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-07T03:01:16.699 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: cluster 2024-03-07T03:01:15.169427+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.318751+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24106 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi148=y;smithi195=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: cephadm 2024-03-07T03:01:15.322331+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi148=y;smithi195=x;count:2 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.328947+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.347607+0000 mon.a (mon.0) 236 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.351351+0000 mon.a (mon.0) 237 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.353139+0000 mon.a (mon.0) 238 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.362389+0000 mon.a (mon.0) 239 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.365476+0000 mon.a (mon.0) 240 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.371845+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.373588+0000 mon.a (mon.0) 242 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: audit 2024-03-07T03:01:15.375463+0000 mon.a (mon.0) 243 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:16.700 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:16 smithi195 bash[23221]: cephadm 2024-03-07T03:01:15.377487+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:01:16.727 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_3 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 916 Links: 1 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-03-07 02:46:38.441377208 +0000 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-03-07 02:46:38.177379563 +0000 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-03-07 02:46:38.177379563 +0000 2024-03-07T03:01:16.774 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: - 2024-03-07T03:01:16.775 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-07T03:01:16.828 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-03-07T03:01:16.828 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-03-07T03:01:16.828 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.00041272 s, 1.2 MB/s 2024-03-07T03:01:16.829 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-07T03:01:16.880 DEBUG:teuthology.orchestra.run.smithi148:> stat /dev/vg_nvme/lv_4 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout:Device: 5h/5d Inode: 937 Links: 1 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout:Access: 2024-03-07 02:46:38.961372569 +0000 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout:Modify: 2024-03-07 02:46:38.689374995 +0000 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout:Change: 2024-03-07 02:46:38.689374995 +0000 2024-03-07T03:01:16.926 INFO:teuthology.orchestra.run.smithi148.stdout: Birth: - 2024-03-07T03:01:16.926 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-07T03:01:16.980 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records in 2024-03-07T03:01:16.980 INFO:teuthology.orchestra.run.smithi148.stderr:1+0 records out 2024-03-07T03:01:16.980 INFO:teuthology.orchestra.run.smithi148.stderr:512 bytes copied, 0.000351603 s, 1.5 MB/s 2024-03-07T03:01:16.981 DEBUG:teuthology.orchestra.run.smithi148:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-07T03:01:17.031 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T03:01:17.031 DEBUG:teuthology.orchestra.run.smithi195:> dd if=/scratch_devs of=/dev/stdout 2024-03-07T03:01:17.038 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-07T03:01:17.038 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_1 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 881 Links: 1 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-03-07 02:46:37.448125003 +0000 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-03-07 02:46:37.160127575 +0000 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-03-07 02:46:37.160127575 +0000 2024-03-07T03:01:17.084 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: - 2024-03-07T03:01:17.085 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-07T03:01:17.138 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-03-07T03:01:17.139 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-03-07T03:01:17.139 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000316167 s, 1.6 MB/s 2024-03-07T03:01:17.140 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-07T03:01:17.190 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_2 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 898 Links: 1 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-03-07 02:46:37.936120645 +0000 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-03-07 02:46:37.648123217 +0000 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-03-07 02:46:37.648123217 +0000 2024-03-07T03:01:17.236 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: - 2024-03-07T03:01:17.236 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-07T03:01:17.290 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-03-07T03:01:17.291 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-03-07T03:01:17.291 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000336555 s, 1.5 MB/s 2024-03-07T03:01:17.292 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-07T03:01:17.341 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_3 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 911 Links: 1 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-03-07 02:46:38.156118680 +0000 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-03-07 02:46:38.152118715 +0000 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-03-07 02:46:38.152118715 +0000 2024-03-07T03:01:17.388 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: - 2024-03-07T03:01:17.388 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-07T03:01:17.443 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-03-07T03:01:17.443 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-03-07T03:01:17.443 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000302275 s, 1.7 MB/s 2024-03-07T03:01:17.444 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-07T03:01:17.493 DEBUG:teuthology.orchestra.run.smithi195:> stat /dev/vg_nvme/lv_4 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout:Device: 5h/5d Inode: 930 Links: 1 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout:Access: 2024-03-07 02:46:38.648114284 +0000 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout:Modify: 2024-03-07 02:46:38.644114320 +0000 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout:Change: 2024-03-07 02:46:38.644114320 +0000 2024-03-07T03:01:17.540 INFO:teuthology.orchestra.run.smithi195.stdout: Birth: - 2024-03-07T03:01:17.540 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-07T03:01:17.594 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records in 2024-03-07T03:01:17.594 INFO:teuthology.orchestra.run.smithi195.stderr:1+0 records out 2024-03-07T03:01:17.594 INFO:teuthology.orchestra.run.smithi195.stderr:512 bytes copied, 0.000184851 s, 2.8 MB/s 2024-03-07T03:01:17.595 DEBUG:teuthology.orchestra.run.smithi195:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-07T03:01:17.645 INFO:tasks.cephadm:Deploying osd.0 on smithi148 with /dev/vg_nvme/lv_4... 2024-03-07T03:01:17.646 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-07T03:01:17.982 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:17.982 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:17.983 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:17.983 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:18.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:18.325 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:18.325 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:18 smithi195 systemd[1]: Started Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:01:19.325 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:18 smithi195 bash[24174]: debug 2024-03-07T03:01:18.982+0000 7f4e13c06700 1 -- 172.21.15.195:0/2959379366 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a9162131e0 con 0x55a916f94400 2024-03-07T03:01:19.325 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[24174]: debug 2024-03-07T03:01:19.126+0000 7f4e1dd68200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:01:19.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[23221]: cluster 2024-03-07T03:01:17.169884+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:19.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[23221]: audit 2024-03-07T03:01:18.126660+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[23221]: audit 2024-03-07T03:01:18.134305+0000 mon.a (mon.0) 245 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[23221]: audit 2024-03-07T03:01:18.154726+0000 mon.a (mon.0) 246 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[23221]: audit 2024-03-07T03:01:18.161635+0000 mon.a (mon.0) 247 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[23221]: audit 2024-03-07T03:01:18.231191+0000 mon.a (mon.0) 248 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:19.326 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:19 smithi195 bash[23221]: audit 2024-03-07T03:01:18.241538+0000 mon.a (mon.0) 249 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[21362]: cluster 2024-03-07T03:01:17.169884+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[21362]: audit 2024-03-07T03:01:18.126660+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[21362]: audit 2024-03-07T03:01:18.134305+0000 mon.a (mon.0) 245 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[21362]: audit 2024-03-07T03:01:18.154726+0000 mon.a (mon.0) 246 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[21362]: audit 2024-03-07T03:01:18.161635+0000 mon.a (mon.0) 247 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[21362]: audit 2024-03-07T03:01:18.231191+0000 mon.a (mon.0) 248 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[21362]: audit 2024-03-07T03:01:18.241538+0000 mon.a (mon.0) 249 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[31149]: cluster 2024-03-07T03:01:17.169884+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[31149]: audit 2024-03-07T03:01:18.126660+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[31149]: audit 2024-03-07T03:01:18.134305+0000 mon.a (mon.0) 245 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[31149]: audit 2024-03-07T03:01:18.154726+0000 mon.a (mon.0) 246 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[31149]: audit 2024-03-07T03:01:18.161635+0000 mon.a (mon.0) 247 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:19.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[31149]: audit 2024-03-07T03:01:18.231191+0000 mon.a (mon.0) 248 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:19.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:19 smithi148 bash[31149]: audit 2024-03-07T03:01:18.241538+0000 mon.a (mon.0) 249 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:21.012 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:20 smithi195 bash[24174]: debug 2024-03-07T03:01:20.666+0000 7f4e1dd68200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:01:21.012 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:20 smithi195 bash[24174]: debug 2024-03-07T03:01:20.762+0000 7f4e1dd68200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:01:21.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:21 smithi195 bash[23221]: cluster 2024-03-07T03:01:19.170302+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:21.325 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:21 smithi195 bash[24174]: debug 2024-03-07T03:01:21.010+0000 7f4e1dd68200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:01:21.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:21 smithi148 bash[21362]: cluster 2024-03-07T03:01:19.170302+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:21.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:21 smithi148 bash[31149]: cluster 2024-03-07T03:01:19.170302+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:22.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:21 smithi195 bash[24174]: debug 2024-03-07T03:01:21.786+0000 7f4e1dd68200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:01:22.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:21 smithi195 bash[24174]: debug 2024-03-07T03:01:21.894+0000 7f4e1dd68200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:01:22.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:21 smithi195 bash[24174]: debug 2024-03-07T03:01:21.994+0000 7f4e1dd68200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:01:22.319 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:01:22.825 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:22 smithi195 bash[24174]: debug 2024-03-07T03:01:22.342+0000 7f4e1dd68200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:01:22.825 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:22 smithi195 bash[24174]: debug 2024-03-07T03:01:22.454+0000 7f4e1dd68200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:01:22.825 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:22 smithi195 bash[24174]: debug 2024-03-07T03:01:22.554+0000 7f4e1dd68200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:01:23.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:23 smithi148 bash[21362]: cluster 2024-03-07T03:01:21.170708+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:23.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:23 smithi148 bash[31149]: cluster 2024-03-07T03:01:21.170708+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:23.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:23 smithi195 bash[23221]: cluster 2024-03-07T03:01:21.170708+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:23.325 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:22 smithi195 bash[24174]: debug 2024-03-07T03:01:22.898+0000 7f4e1dd68200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:01:23.325 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:23 smithi195 bash[24174]: debug 2024-03-07T03:01:23.142+0000 7f4e1dd68200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:01:23.792 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:23 smithi195 bash[24174]: debug 2024-03-07T03:01:23.402+0000 7f4e1dd68200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:01:23.792 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:23 smithi195 bash[24174]: debug 2024-03-07T03:01:23.518+0000 7f4e1dd68200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:01:24.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:23 smithi195 bash[24174]: debug 2024-03-07T03:01:23.790+0000 7f4e1dd68200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:01:24.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[24174]: debug 2024-03-07T03:01:24.026+0000 7f4e1dd68200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:01:24.575 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[24174]: debug 2024-03-07T03:01:24.430+0000 7f4e1dd68200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:01:24.575 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[24174]: debug 2024-03-07T03:01:24.546+0000 7f4e1dd68200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:01:24.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: cluster 2024-03-07T03:01:23.171069+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:24.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.317853+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.323583+0000 mon.a (mon.0) 251 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.793559+0000 mon.a (mon.0) 252 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:24.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.794412+0000 mon.a (mon.0) 253 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:24.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.799630+0000 mon.a (mon.0) 254 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: cephadm 2024-03-07T03:01:23.849882+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-03-07T03:01:24.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.850419+0000 mon.a (mon.0) 255 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:01:24.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.851403+0000 mon.a (mon.0) 256 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:01:24.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: audit 2024-03-07T03:01:23.852126+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:24.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[23221]: cephadm 2024-03-07T03:01:23.853120+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi148 2024-03-07T03:01:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: cluster 2024-03-07T03:01:23.171069+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.317853+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.323583+0000 mon.a (mon.0) 251 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.793559+0000 mon.a (mon.0) 252 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.794412+0000 mon.a (mon.0) 253 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.799630+0000 mon.a (mon.0) 254 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: cephadm 2024-03-07T03:01:23.849882+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.850419+0000 mon.a (mon.0) 255 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.851403+0000 mon.a (mon.0) 256 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: audit 2024-03-07T03:01:23.852126+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[31149]: cephadm 2024-03-07T03:01:23.853120+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi148 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: cluster 2024-03-07T03:01:23.171069+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.317853+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.323583+0000 mon.a (mon.0) 251 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.793559+0000 mon.a (mon.0) 252 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.794412+0000 mon.a (mon.0) 253 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.799630+0000 mon.a (mon.0) 254 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: cephadm 2024-03-07T03:01:23.849882+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-03-07T03:01:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.850419+0000 mon.a (mon.0) 255 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:01:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.851403+0000 mon.a (mon.0) 256 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:01:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: audit 2024-03-07T03:01:23.852126+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:24 smithi148 bash[21362]: cephadm 2024-03-07T03:01:23.853120+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi148 2024-03-07T03:01:25.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:24 smithi195 bash[24174]: debug 2024-03-07T03:01:24.782+0000 7f4e1dd68200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:01:25.464 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-07T03:01:25.465 INFO:teuthology.orchestra.run.smithi148.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-07T03:01:25.465 INFO:teuthology.orchestra.run.smithi148.stderr: stderr: 10+0 records in 2024-03-07T03:01:25.465 INFO:teuthology.orchestra.run.smithi148.stderr:10+0 records out 2024-03-07T03:01:25.465 INFO:teuthology.orchestra.run.smithi148.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0151977 s, 690 MB/s 2024-03-07T03:01:25.465 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping successful for: 2024-03-07T03:01:26.074 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:25 smithi195 bash[24174]: debug 2024-03-07T03:01:25.770+0000 7f4e1dd68200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:01:26.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:25 smithi195 bash[24174]: debug 2024-03-07T03:01:25.886+0000 7f4e1dd68200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:01:26.198 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi148:vg_nvme/lv_4 2024-03-07T03:01:26.575 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[24174]: debug 2024-03-07T03:01:26.170+0000 7f4e1dd68200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:01:26.575 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[24174]: debug 2024-03-07T03:01:26.278+0000 7f4e1dd68200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[24174]: debug 2024-03-07T03:01:26.718+0000 7f4e1dd68200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[24174]: debug 2024-03-07T03:01:26.834+0000 7f4e1dd68200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[23221]: cluster 2024-03-07T03:01:25.171432+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[23221]: audit 2024-03-07T03:01:25.807939+0000 mon.a (mon.0) 258 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[23221]: audit 2024-03-07T03:01:25.815970+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[23221]: audit 2024-03-07T03:01:25.821940+0000 mon.a (mon.0) 260 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[23221]: audit 2024-03-07T03:01:25.824374+0000 mon.a (mon.0) 261 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[23221]: audit 2024-03-07T03:01:25.825685+0000 mon.a (mon.0) 262 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:27.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:26 smithi195 bash[23221]: audit 2024-03-07T03:01:25.834242+0000 mon.a (mon.0) 263 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[21362]: cluster 2024-03-07T03:01:25.171432+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[21362]: audit 2024-03-07T03:01:25.807939+0000 mon.a (mon.0) 258 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[21362]: audit 2024-03-07T03:01:25.815970+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[21362]: audit 2024-03-07T03:01:25.821940+0000 mon.a (mon.0) 260 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[21362]: audit 2024-03-07T03:01:25.824374+0000 mon.a (mon.0) 261 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[21362]: audit 2024-03-07T03:01:25.825685+0000 mon.a (mon.0) 262 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[21362]: audit 2024-03-07T03:01:25.834242+0000 mon.a (mon.0) 263 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[31149]: cluster 2024-03-07T03:01:25.171432+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[31149]: audit 2024-03-07T03:01:25.807939+0000 mon.a (mon.0) 258 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[31149]: audit 2024-03-07T03:01:25.815970+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[31149]: audit 2024-03-07T03:01:25.821940+0000 mon.a (mon.0) 260 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[31149]: audit 2024-03-07T03:01:25.824374+0000 mon.a (mon.0) 261 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:27.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[31149]: audit 2024-03-07T03:01:25.825685+0000 mon.a (mon.0) 262 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:01:27.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:26 smithi148 bash[31149]: audit 2024-03-07T03:01:25.834242+0000 mon.a (mon.0) 263 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:28.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:27 smithi195 bash[23221]: audit 2024-03-07T03:01:26.865592+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:01:28.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:27 smithi195 bash[23221]: cluster 2024-03-07T03:01:26.865811+0000 mon.a (mon.0) 264 : cluster 0 Standby manager daemon x started 2024-03-07T03:01:28.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:27 smithi195 bash[23221]: audit 2024-03-07T03:01:26.866710+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:01:28.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:27 smithi195 bash[23221]: audit 2024-03-07T03:01:26.867838+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:01:28.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:27 smithi195 bash[23221]: audit 2024-03-07T03:01:26.868279+0000 mon.b (mon.1) 7 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:01:28.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[21362]: audit 2024-03-07T03:01:26.865592+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[21362]: cluster 2024-03-07T03:01:26.865811+0000 mon.a (mon.0) 264 : cluster 0 Standby manager daemon x started 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[21362]: audit 2024-03-07T03:01:26.866710+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[21362]: audit 2024-03-07T03:01:26.867838+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[21362]: audit 2024-03-07T03:01:26.868279+0000 mon.b (mon.1) 7 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[31149]: audit 2024-03-07T03:01:26.865592+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[31149]: cluster 2024-03-07T03:01:26.865811+0000 mon.a (mon.0) 264 : cluster 0 Standby manager daemon x started 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[31149]: audit 2024-03-07T03:01:26.866710+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[31149]: audit 2024-03-07T03:01:26.867838+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:01:28.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:27 smithi148 bash[31149]: audit 2024-03-07T03:01:26.868279+0000 mon.b (mon.1) 7 : audit 0 from='mgr.? 172.21.15.195:0/1460493155' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:01:29.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:28 smithi195 bash[23221]: cluster 2024-03-07T03:01:27.171854+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:29.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:28 smithi195 bash[23221]: cluster 2024-03-07T03:01:27.829281+0000 mon.a (mon.0) 265 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-03-07T03:01:29.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:28 smithi195 bash[23221]: audit 2024-03-07T03:01:27.829553+0000 mon.a (mon.0) 266 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:01:29.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:28 smithi148 bash[21362]: cluster 2024-03-07T03:01:27.171854+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:29.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:28 smithi148 bash[21362]: cluster 2024-03-07T03:01:27.829281+0000 mon.a (mon.0) 265 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-03-07T03:01:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:28 smithi148 bash[21362]: audit 2024-03-07T03:01:27.829553+0000 mon.a (mon.0) 266 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:01:29.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:28 smithi148 bash[31149]: cluster 2024-03-07T03:01:27.171854+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:29.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:28 smithi148 bash[31149]: cluster 2024-03-07T03:01:27.829281+0000 mon.a (mon.0) 265 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-03-07T03:01:29.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:28 smithi148 bash[31149]: audit 2024-03-07T03:01:27.829553+0000 mon.a (mon.0) 266 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:01:30.873 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:01:31.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:30 smithi195 bash[23221]: cluster 2024-03-07T03:01:29.172399+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:31.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:30 smithi148 bash[31149]: cluster 2024-03-07T03:01:29.172399+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:31.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:30 smithi148 bash[21362]: cluster 2024-03-07T03:01:29.172399+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:33.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:32 smithi148 bash[31149]: cluster 2024-03-07T03:01:31.172803+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:33.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:32 smithi148 bash[21362]: cluster 2024-03-07T03:01:31.172803+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:33.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:32 smithi195 bash[23221]: cluster 2024-03-07T03:01:31.172803+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:34.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:34 smithi195 bash[23221]: audit 2024-03-07T03:01:33.267531+0000 mon.a (mon.0) 267 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:01:34.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:34 smithi195 bash[23221]: audit 2024-03-07T03:01:33.271360+0000 mon.a (mon.0) 268 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:01:34.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:34 smithi195 bash[23221]: audit 2024-03-07T03:01:33.272536+0000 mon.a (mon.0) 269 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:34.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:34 smithi148 bash[21362]: audit 2024-03-07T03:01:33.267531+0000 mon.a (mon.0) 267 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:01:34.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:34 smithi148 bash[21362]: audit 2024-03-07T03:01:33.271360+0000 mon.a (mon.0) 268 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:01:34.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:34 smithi148 bash[21362]: audit 2024-03-07T03:01:33.272536+0000 mon.a (mon.0) 269 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:34.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:34 smithi148 bash[31149]: audit 2024-03-07T03:01:33.267531+0000 mon.a (mon.0) 267 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:01:34.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:34 smithi148 bash[31149]: audit 2024-03-07T03:01:33.271360+0000 mon.a (mon.0) 268 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:01:34.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:34 smithi148 bash[31149]: audit 2024-03-07T03:01:33.272536+0000 mon.a (mon.0) 269 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:35.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:35 smithi195 bash[23221]: cluster 2024-03-07T03:01:33.173281+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:35.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:35 smithi195 bash[23221]: audit 2024-03-07T03:01:33.260388+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:35.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:35 smithi148 bash[21362]: cluster 2024-03-07T03:01:33.173281+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:35.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:35 smithi148 bash[21362]: audit 2024-03-07T03:01:33.260388+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:35.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:35 smithi148 bash[31149]: cluster 2024-03-07T03:01:33.173281+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:35.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:35 smithi148 bash[31149]: audit 2024-03-07T03:01:33.260388+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.24110 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:01:37.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:37 smithi195 bash[23221]: cluster 2024-03-07T03:01:35.173822+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:37.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:37 smithi148 bash[21362]: cluster 2024-03-07T03:01:35.173822+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:37.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:37 smithi148 bash[31149]: cluster 2024-03-07T03:01:35.173822+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:39.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:39 smithi195 bash[23221]: cluster 2024-03-07T03:01:37.174260+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:39.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:39 smithi148 bash[21362]: cluster 2024-03-07T03:01:37.174260+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:39.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:39 smithi148 bash[31149]: cluster 2024-03-07T03:01:37.174260+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:41.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:41 smithi195 bash[23221]: cluster 2024-03-07T03:01:39.174680+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:41.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:41 smithi148 bash[21362]: cluster 2024-03-07T03:01:39.174680+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:41 smithi148 bash[31149]: cluster 2024-03-07T03:01:39.174680+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:42.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:42 smithi195 bash[23221]: audit 2024-03-07T03:01:41.166815+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.148:0/3189079718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]: dispatch 2024-03-07T03:01:42.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:42 smithi195 bash[23221]: audit 2024-03-07T03:01:41.167386+0000 mon.a (mon.0) 270 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]: dispatch 2024-03-07T03:01:42.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:42 smithi195 bash[23221]: audit 2024-03-07T03:01:41.174181+0000 mon.a (mon.0) 271 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]': finished 2024-03-07T03:01:42.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:42 smithi195 bash[23221]: cluster 2024-03-07T03:01:41.174297+0000 mon.a (mon.0) 272 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-03-07T03:01:42.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:42 smithi195 bash[23221]: audit 2024-03-07T03:01:41.174412+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:01:42.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:42 smithi195 bash[23221]: audit 2024-03-07T03:01:41.874464+0000 mon.a (mon.0) 274 : audit 0 from='client.? 172.21.15.148:0/1190113482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:01:42.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[21362]: audit 2024-03-07T03:01:41.166815+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.148:0/3189079718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]: dispatch 2024-03-07T03:01:42.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[21362]: audit 2024-03-07T03:01:41.167386+0000 mon.a (mon.0) 270 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]: dispatch 2024-03-07T03:01:42.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[21362]: audit 2024-03-07T03:01:41.174181+0000 mon.a (mon.0) 271 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]': finished 2024-03-07T03:01:42.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[21362]: cluster 2024-03-07T03:01:41.174297+0000 mon.a (mon.0) 272 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-03-07T03:01:42.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[21362]: audit 2024-03-07T03:01:41.174412+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:01:42.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[21362]: audit 2024-03-07T03:01:41.874464+0000 mon.a (mon.0) 274 : audit 0 from='client.? 172.21.15.148:0/1190113482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:01:42.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[31149]: audit 2024-03-07T03:01:41.166815+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.148:0/3189079718' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]: dispatch 2024-03-07T03:01:42.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[31149]: audit 2024-03-07T03:01:41.167386+0000 mon.a (mon.0) 270 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]: dispatch 2024-03-07T03:01:42.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[31149]: audit 2024-03-07T03:01:41.174181+0000 mon.a (mon.0) 271 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6f1dd05f-29db-4a04-8e52-e0c19aeaf719"}]': finished 2024-03-07T03:01:42.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[31149]: cluster 2024-03-07T03:01:41.174297+0000 mon.a (mon.0) 272 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-03-07T03:01:42.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[31149]: audit 2024-03-07T03:01:41.174412+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:01:42.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:42 smithi148 bash[31149]: audit 2024-03-07T03:01:41.874464+0000 mon.a (mon.0) 274 : audit 0 from='client.? 172.21.15.148:0/1190113482' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:01:43.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:43 smithi148 bash[21362]: cluster 2024-03-07T03:01:41.174915+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:43.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:43 smithi148 bash[31149]: cluster 2024-03-07T03:01:41.174915+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:43.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:43 smithi195 bash[23221]: cluster 2024-03-07T03:01:41.174915+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:44.499 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:44 smithi148 bash[21362]: cluster 2024-03-07T03:01:43.175307+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:44.499 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:44 smithi148 bash[31149]: cluster 2024-03-07T03:01:43.175307+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:44.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:44 smithi195 bash[23221]: cluster 2024-03-07T03:01:43.175307+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:46.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:46 smithi148 bash[21362]: cluster 2024-03-07T03:01:45.175738+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:46.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:46 smithi148 bash[31149]: cluster 2024-03-07T03:01:45.175738+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:46.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:46 smithi195 bash[23221]: cluster 2024-03-07T03:01:45.175738+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:49.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:49 smithi195 bash[23221]: cluster 2024-03-07T03:01:47.176079+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:49.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:49 smithi148 bash[21362]: cluster 2024-03-07T03:01:47.176079+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:49.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:49 smithi148 bash[31149]: cluster 2024-03-07T03:01:47.176079+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:51.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:51 smithi148 bash[21362]: cluster 2024-03-07T03:01:49.176464+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:51.174 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:51 smithi148 bash[31149]: cluster 2024-03-07T03:01:49.176464+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:51.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:51 smithi195 bash[23221]: cluster 2024-03-07T03:01:49.176464+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:53.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:53 smithi195 bash[23221]: cluster 2024-03-07T03:01:51.176744+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:53.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:53 smithi195 bash[23221]: audit 2024-03-07T03:01:52.900341+0000 mon.a (mon.0) 275 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:01:53.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:53 smithi195 bash[23221]: audit 2024-03-07T03:01:52.902058+0000 mon.a (mon.0) 276 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:53.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:53 smithi148 bash[21362]: cluster 2024-03-07T03:01:51.176744+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:53.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:53 smithi148 bash[21362]: audit 2024-03-07T03:01:52.900341+0000 mon.a (mon.0) 275 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:01:53.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:53 smithi148 bash[21362]: audit 2024-03-07T03:01:52.902058+0000 mon.a (mon.0) 276 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:53 smithi148 bash[31149]: cluster 2024-03-07T03:01:51.176744+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:53 smithi148 bash[31149]: audit 2024-03-07T03:01:52.900341+0000 mon.a (mon.0) 275 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:01:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:53 smithi148 bash[31149]: audit 2024-03-07T03:01:52.902058+0000 mon.a (mon.0) 276 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:01:54.224 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:54 smithi148 bash[21362]: cephadm 2024-03-07T03:01:52.903564+0000 mgr.y (mgr.14152) 97 : cephadm 1 Deploying daemon osd.0 on smithi148 2024-03-07T03:01:54.224 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:54 smithi148 bash[31149]: cephadm 2024-03-07T03:01:52.903564+0000 mgr.y (mgr.14152) 97 : cephadm 1 Deploying daemon osd.0 on smithi148 2024-03-07T03:01:54.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:54 smithi195 bash[23221]: cephadm 2024-03-07T03:01:52.903564+0000 mgr.y (mgr.14152) 97 : cephadm 1 Deploying daemon osd.0 on smithi148 2024-03-07T03:01:55.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:55 smithi195 bash[23221]: cluster 2024-03-07T03:01:53.177146+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:55.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:55 smithi148 bash[31149]: cluster 2024-03-07T03:01:53.177146+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:55.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:55 smithi148 bash[21362]: cluster 2024-03-07T03:01:53.177146+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:56.503 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:56 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:56.503 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:56 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:56.503 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:01:56 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:56.768 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:56 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:56.769 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:56 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:56.769 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:01:56 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:01:57.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:57 smithi195 bash[23221]: cluster 2024-03-07T03:01:55.177611+0000 mgr.y (mgr.14152) 99 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:57.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:57 smithi195 bash[23221]: audit 2024-03-07T03:01:56.757469+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:57.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:57 smithi195 bash[23221]: audit 2024-03-07T03:01:56.761906+0000 mon.a (mon.0) 278 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:57.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:57 smithi195 bash[23221]: audit 2024-03-07T03:01:56.788238+0000 mon.a (mon.0) 279 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[21362]: cluster 2024-03-07T03:01:55.177611+0000 mgr.y (mgr.14152) 99 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[21362]: audit 2024-03-07T03:01:56.757469+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[21362]: audit 2024-03-07T03:01:56.761906+0000 mon.a (mon.0) 278 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[21362]: audit 2024-03-07T03:01:56.788238+0000 mon.a (mon.0) 279 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[31149]: cluster 2024-03-07T03:01:55.177611+0000 mgr.y (mgr.14152) 99 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[31149]: audit 2024-03-07T03:01:56.757469+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[31149]: audit 2024-03-07T03:01:56.761906+0000 mon.a (mon.0) 278 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:01:57.534 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:57 smithi148 bash[31149]: audit 2024-03-07T03:01:56.788238+0000 mon.a (mon.0) 279 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:01:59.327 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:01:59 smithi148 bash[21362]: cluster 2024-03-07T03:01:57.177959+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:59.328 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:01:59 smithi148 bash[31149]: cluster 2024-03-07T03:01:57.177959+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:01:59.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:01:59 smithi195 bash[23221]: cluster 2024-03-07T03:01:57.177959+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:01.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:01 smithi148 bash[21362]: cluster 2024-03-07T03:01:59.178352+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:01.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:01 smithi148 bash[31149]: cluster 2024-03-07T03:01:59.178352+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:01.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:01 smithi195 bash[23221]: cluster 2024-03-07T03:01:59.178352+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[21362]: cluster 2024-03-07T03:02:01.178798+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[21362]: audit 2024-03-07T03:02:01.871140+0000 mon.a (mon.0) 280 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[21362]: audit 2024-03-07T03:02:01.876128+0000 mon.a (mon.0) 281 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[21362]: audit 2024-03-07T03:02:02.298981+0000 mon.a (mon.0) 282 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[21362]: audit 2024-03-07T03:02:02.299766+0000 mon.a (mon.0) 283 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[21362]: audit 2024-03-07T03:02:02.305744+0000 mon.a (mon.0) 284 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[31149]: cluster 2024-03-07T03:02:01.178798+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[31149]: audit 2024-03-07T03:02:01.871140+0000 mon.a (mon.0) 280 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[31149]: audit 2024-03-07T03:02:01.876128+0000 mon.a (mon.0) 281 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[31149]: audit 2024-03-07T03:02:02.298981+0000 mon.a (mon.0) 282 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[31149]: audit 2024-03-07T03:02:02.299766+0000 mon.a (mon.0) 283 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:02 smithi148 bash[31149]: audit 2024-03-07T03:02:02.305744+0000 mon.a (mon.0) 284 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:02 smithi195 bash[23221]: cluster 2024-03-07T03:02:01.178798+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:03.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:02 smithi195 bash[23221]: audit 2024-03-07T03:02:01.871140+0000 mon.a (mon.0) 280 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:02 smithi195 bash[23221]: audit 2024-03-07T03:02:01.876128+0000 mon.a (mon.0) 281 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:03.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:02 smithi195 bash[23221]: audit 2024-03-07T03:02:02.298981+0000 mon.a (mon.0) 282 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:03.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:02 smithi195 bash[23221]: audit 2024-03-07T03:02:02.299766+0000 mon.a (mon.0) 283 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:03.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:02 smithi195 bash[23221]: audit 2024-03-07T03:02:02.305744+0000 mon.a (mon.0) 284 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:04.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:03 smithi148 bash[31149]: audit 2024-03-07T03:02:02.936558+0000 mon.c (mon.2) 4 : audit 1 from='osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:02:04.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:03 smithi148 bash[31149]: audit 2024-03-07T03:02:02.937006+0000 mon.a (mon.0) 285 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:02:04.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:03 smithi148 bash[21362]: audit 2024-03-07T03:02:02.936558+0000 mon.c (mon.2) 4 : audit 1 from='osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:02:04.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:03 smithi148 bash[21362]: audit 2024-03-07T03:02:02.937006+0000 mon.a (mon.0) 285 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:02:04.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:03 smithi195 bash[23221]: audit 2024-03-07T03:02:02.936558+0000 mon.c (mon.2) 4 : audit 1 from='osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:02:04.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:03 smithi195 bash[23221]: audit 2024-03-07T03:02:02.937006+0000 mon.a (mon.0) 285 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:02:04.707 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 0 on host 'smithi148' 2024-03-07T03:02:05.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: cluster 2024-03-07T03:02:03.179275+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:05.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: audit 2024-03-07T03:02:03.885007+0000 mon.a (mon.0) 286 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:02:05.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: cluster 2024-03-07T03:02:03.885209+0000 mon.a (mon.0) 287 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-03-07T03:02:05.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: audit 2024-03-07T03:02:03.885409+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:05.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: audit 2024-03-07T03:02:03.886243+0000 mon.c (mon.2) 5 : audit 1 from='osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: audit 2024-03-07T03:02:03.886595+0000 mon.a (mon.0) 289 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: audit 2024-03-07T03:02:04.683424+0000 mon.a (mon.0) 290 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: audit 2024-03-07T03:02:04.692193+0000 mon.a (mon.0) 291 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[21362]: audit 2024-03-07T03:02:04.724513+0000 mon.a (mon.0) 292 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: cluster 2024-03-07T03:02:03.179275+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: audit 2024-03-07T03:02:03.885007+0000 mon.a (mon.0) 286 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: cluster 2024-03-07T03:02:03.885209+0000 mon.a (mon.0) 287 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: audit 2024-03-07T03:02:03.885409+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:05.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: audit 2024-03-07T03:02:03.886243+0000 mon.c (mon.2) 5 : audit 1 from='osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:05.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: audit 2024-03-07T03:02:03.886595+0000 mon.a (mon.0) 289 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:05.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: audit 2024-03-07T03:02:04.683424+0000 mon.a (mon.0) 290 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:05.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: audit 2024-03-07T03:02:04.692193+0000 mon.a (mon.0) 291 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:05.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:04 smithi148 bash[31149]: audit 2024-03-07T03:02:04.724513+0000 mon.a (mon.0) 292 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:05.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: cluster 2024-03-07T03:02:03.179275+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: audit 2024-03-07T03:02:03.885007+0000 mon.a (mon.0) 286 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: cluster 2024-03-07T03:02:03.885209+0000 mon.a (mon.0) 287 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: audit 2024-03-07T03:02:03.885409+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: audit 2024-03-07T03:02:03.886243+0000 mon.c (mon.2) 5 : audit 1 from='osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: audit 2024-03-07T03:02:03.886595+0000 mon.a (mon.0) 289 : audit 1 from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: audit 2024-03-07T03:02:04.683424+0000 mon.a (mon.0) 290 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: audit 2024-03-07T03:02:04.692193+0000 mon.a (mon.0) 291 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:05.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:04 smithi195 bash[23221]: audit 2024-03-07T03:02:04.724513+0000 mon.a (mon.0) 292 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:05.356 DEBUG:teuthology.orchestra.run.smithi148:osd.0> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.0.service 2024-03-07T03:02:05.358 INFO:tasks.cephadm:Deploying osd.1 on smithi148 with /dev/vg_nvme/lv_3... 2024-03-07T03:02:05.359 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-07T03:02:06.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[21362]: audit 2024-03-07T03:02:04.892896+0000 mon.a (mon.0) 293 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:02:06.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[21362]: cluster 2024-03-07T03:02:04.893048+0000 mon.a (mon.0) 294 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-03-07T03:02:06.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[21362]: audit 2024-03-07T03:02:04.893679+0000 mon.a (mon.0) 295 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:06.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[21362]: audit 2024-03-07T03:02:04.901866+0000 mon.a (mon.0) 296 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:06.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[31149]: audit 2024-03-07T03:02:04.892896+0000 mon.a (mon.0) 293 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:02:06.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[31149]: cluster 2024-03-07T03:02:04.893048+0000 mon.a (mon.0) 294 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-03-07T03:02:06.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[31149]: audit 2024-03-07T03:02:04.893679+0000 mon.a (mon.0) 295 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:06.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:05 smithi148 bash[31149]: audit 2024-03-07T03:02:04.901866+0000 mon.a (mon.0) 296 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:06.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:05 smithi195 bash[23221]: audit 2024-03-07T03:02:04.892896+0000 mon.a (mon.0) 293 : audit 1 from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:02:06.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:05 smithi195 bash[23221]: cluster 2024-03-07T03:02:04.893048+0000 mon.a (mon.0) 294 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-03-07T03:02:06.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:05 smithi195 bash[23221]: audit 2024-03-07T03:02:04.893679+0000 mon.a (mon.0) 295 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:06.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:05 smithi195 bash[23221]: audit 2024-03-07T03:02:04.901866+0000 mon.a (mon.0) 296 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:07.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[21362]: cluster 2024-03-07T03:02:03.962306+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:07.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[21362]: cluster 2024-03-07T03:02:03.962365+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:07.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[21362]: cluster 2024-03-07T03:02:05.179730+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:07.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[21362]: cluster 2024-03-07T03:02:05.909011+0000 mon.a (mon.0) 297 : cluster 1 osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746] boot 2024-03-07T03:02:07.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[21362]: cluster 2024-03-07T03:02:05.909075+0000 mon.a (mon.0) 298 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-03-07T03:02:07.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[21362]: audit 2024-03-07T03:02:05.909726+0000 mon.a (mon.0) 299 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:07.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[31149]: cluster 2024-03-07T03:02:03.962306+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:07.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[31149]: cluster 2024-03-07T03:02:03.962365+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:07.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[31149]: cluster 2024-03-07T03:02:05.179730+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:07.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[31149]: cluster 2024-03-07T03:02:05.909011+0000 mon.a (mon.0) 297 : cluster 1 osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746] boot 2024-03-07T03:02:07.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[31149]: cluster 2024-03-07T03:02:05.909075+0000 mon.a (mon.0) 298 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-03-07T03:02:07.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:06 smithi148 bash[31149]: audit 2024-03-07T03:02:05.909726+0000 mon.a (mon.0) 299 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:07.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:06 smithi195 bash[23221]: cluster 2024-03-07T03:02:03.962306+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:07.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:06 smithi195 bash[23221]: cluster 2024-03-07T03:02:03.962365+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:07.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:06 smithi195 bash[23221]: cluster 2024-03-07T03:02:05.179730+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-07T03:02:07.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:06 smithi195 bash[23221]: cluster 2024-03-07T03:02:05.909011+0000 mon.a (mon.0) 297 : cluster 1 osd.0 [v2:172.21.15.148:6802/1285694746,v1:172.21.15.148:6803/1285694746] boot 2024-03-07T03:02:07.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:06 smithi195 bash[23221]: cluster 2024-03-07T03:02:05.909075+0000 mon.a (mon.0) 298 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-03-07T03:02:07.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:06 smithi195 bash[23221]: audit 2024-03-07T03:02:05.909726+0000 mon.a (mon.0) 299 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:02:08.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:07 smithi148 bash[21362]: cluster 2024-03-07T03:02:06.919995+0000 mon.a (mon.0) 300 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-03-07T03:02:08.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:07 smithi148 bash[31149]: cluster 2024-03-07T03:02:06.919995+0000 mon.a (mon.0) 300 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-03-07T03:02:08.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:07 smithi195 bash[23221]: cluster 2024-03-07T03:02:06.919995+0000 mon.a (mon.0) 300 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-03-07T03:02:09.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:09 smithi195 bash[23221]: cluster 2024-03-07T03:02:07.180249+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v71: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:09.542 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:02:09.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:09 smithi148 bash[21362]: cluster 2024-03-07T03:02:07.180249+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v71: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:09.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:09 smithi148 bash[31149]: cluster 2024-03-07T03:02:07.180249+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v71: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:11.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:11 smithi195 bash[23221]: cluster 2024-03-07T03:02:09.180661+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v72: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:11.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:11 smithi148 bash[21362]: cluster 2024-03-07T03:02:09.180661+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v72: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:11.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:11 smithi148 bash[31149]: cluster 2024-03-07T03:02:09.180661+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v72: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:13.163 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-07T03:02:13.163 INFO:teuthology.orchestra.run.smithi148.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-07T03:02:13.163 INFO:teuthology.orchestra.run.smithi148.stderr: stderr: 10+0 records in 2024-03-07T03:02:13.163 INFO:teuthology.orchestra.run.smithi148.stderr:10+0 records out 2024-03-07T03:02:13.163 INFO:teuthology.orchestra.run.smithi148.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0179362 s, 585 MB/s 2024-03-07T03:02:13.163 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping successful for: 2024-03-07T03:02:13.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:13 smithi195 bash[23221]: cluster 2024-03-07T03:02:11.181016+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:13.360 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:13 smithi148 bash[21362]: cluster 2024-03-07T03:02:11.181016+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:13.360 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:13 smithi148 bash[31149]: cluster 2024-03-07T03:02:11.181016+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:13.779 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi148:vg_nvme/lv_3 2024-03-07T03:02:15.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: cluster 2024-03-07T03:02:13.181289+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: audit 2024-03-07T03:02:14.423702+0000 mon.a (mon.0) 301 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: audit 2024-03-07T03:02:14.433038+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: audit 2024-03-07T03:02:14.435238+0000 mon.a (mon.0) 303 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: audit 2024-03-07T03:02:14.445221+0000 mon.a (mon.0) 304 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: audit 2024-03-07T03:02:14.447588+0000 mon.a (mon.0) 305 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: audit 2024-03-07T03:02:14.449499+0000 mon.a (mon.0) 306 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:15.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:15 smithi195 bash[23221]: audit 2024-03-07T03:02:14.459226+0000 mon.a (mon.0) 307 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: cluster 2024-03-07T03:02:13.181289+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: audit 2024-03-07T03:02:14.423702+0000 mon.a (mon.0) 301 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: audit 2024-03-07T03:02:14.433038+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: audit 2024-03-07T03:02:14.435238+0000 mon.a (mon.0) 303 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: audit 2024-03-07T03:02:14.445221+0000 mon.a (mon.0) 304 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: audit 2024-03-07T03:02:14.447588+0000 mon.a (mon.0) 305 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: audit 2024-03-07T03:02:14.449499+0000 mon.a (mon.0) 306 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:15.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[21362]: audit 2024-03-07T03:02:14.459226+0000 mon.a (mon.0) 307 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: cluster 2024-03-07T03:02:13.181289+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: audit 2024-03-07T03:02:14.423702+0000 mon.a (mon.0) 301 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: audit 2024-03-07T03:02:14.433038+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: audit 2024-03-07T03:02:14.435238+0000 mon.a (mon.0) 303 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: audit 2024-03-07T03:02:14.445221+0000 mon.a (mon.0) 304 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: audit 2024-03-07T03:02:14.447588+0000 mon.a (mon.0) 305 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: audit 2024-03-07T03:02:14.449499+0000 mon.a (mon.0) 306 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:15.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:15 smithi148 bash[31149]: audit 2024-03-07T03:02:14.459226+0000 mon.a (mon.0) 307 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:16.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:16 smithi148 bash[21362]: cephadm 2024-03-07T03:02:14.414811+0000 mgr.y (mgr.14152) 109 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:02:16.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:16 smithi148 bash[21362]: cephadm 2024-03-07T03:02:14.436780+0000 mgr.y (mgr.14152) 110 : cephadm 1 Adjusting osd_memory_target on smithi148 to 16241M 2024-03-07T03:02:16.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:16 smithi148 bash[31149]: cephadm 2024-03-07T03:02:14.414811+0000 mgr.y (mgr.14152) 109 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:02:16.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:16 smithi148 bash[31149]: cephadm 2024-03-07T03:02:14.436780+0000 mgr.y (mgr.14152) 110 : cephadm 1 Adjusting osd_memory_target on smithi148 to 16241M 2024-03-07T03:02:16.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:16 smithi195 bash[23221]: cephadm 2024-03-07T03:02:14.414811+0000 mgr.y (mgr.14152) 109 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:02:16.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:16 smithi195 bash[23221]: cephadm 2024-03-07T03:02:14.436780+0000 mgr.y (mgr.14152) 110 : cephadm 1 Adjusting osd_memory_target on smithi148 to 16241M 2024-03-07T03:02:17.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:17 smithi148 bash[21362]: cluster 2024-03-07T03:02:15.181700+0000 mgr.y (mgr.14152) 111 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:17.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:17 smithi148 bash[31149]: cluster 2024-03-07T03:02:15.181700+0000 mgr.y (mgr.14152) 111 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:17.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:17 smithi195 bash[23221]: cluster 2024-03-07T03:02:15.181700+0000 mgr.y (mgr.14152) 111 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:18.455 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:02:19.337 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:19 smithi148 bash[21362]: cluster 2024-03-07T03:02:17.182159+0000 mgr.y (mgr.14152) 112 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:19.338 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:19 smithi148 bash[31149]: cluster 2024-03-07T03:02:17.182159+0000 mgr.y (mgr.14152) 112 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:19.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:19 smithi195 bash[23221]: cluster 2024-03-07T03:02:17.182159+0000 mgr.y (mgr.14152) 112 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[21362]: cluster 2024-03-07T03:02:19.182609+0000 mgr.y (mgr.14152) 113 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[21362]: audit 2024-03-07T03:02:20.909507+0000 mon.a (mon.0) 308 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[21362]: audit 2024-03-07T03:02:20.912474+0000 mon.a (mon.0) 309 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[21362]: audit 2024-03-07T03:02:20.913477+0000 mon.a (mon.0) 310 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[31149]: cluster 2024-03-07T03:02:19.182609+0000 mgr.y (mgr.14152) 113 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[31149]: audit 2024-03-07T03:02:20.909507+0000 mon.a (mon.0) 308 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[31149]: audit 2024-03-07T03:02:20.912474+0000 mon.a (mon.0) 309 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:21.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:21 smithi148 bash[31149]: audit 2024-03-07T03:02:20.913477+0000 mon.a (mon.0) 310 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:21.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:21 smithi195 bash[23221]: cluster 2024-03-07T03:02:19.182609+0000 mgr.y (mgr.14152) 113 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:21.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:21 smithi195 bash[23221]: audit 2024-03-07T03:02:20.909507+0000 mon.a (mon.0) 308 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:02:21.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:21 smithi195 bash[23221]: audit 2024-03-07T03:02:20.912474+0000 mon.a (mon.0) 309 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:02:21.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:21 smithi195 bash[23221]: audit 2024-03-07T03:02:20.913477+0000 mon.a (mon.0) 310 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:22.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:22 smithi148 bash[31149]: audit 2024-03-07T03:02:20.902685+0000 mgr.y (mgr.14152) 114 : audit 0 from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:22.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:22 smithi148 bash[21362]: audit 2024-03-07T03:02:20.902685+0000 mgr.y (mgr.14152) 114 : audit 0 from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:22.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:22 smithi195 bash[23221]: audit 2024-03-07T03:02:20.902685+0000 mgr.y (mgr.14152) 114 : audit 0 from='client.14229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:02:23.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:23 smithi148 bash[21362]: cluster 2024-03-07T03:02:21.182897+0000 mgr.y (mgr.14152) 115 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:23.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:23 smithi148 bash[31149]: cluster 2024-03-07T03:02:21.182897+0000 mgr.y (mgr.14152) 115 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:23.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:23 smithi195 bash[23221]: cluster 2024-03-07T03:02:21.182897+0000 mgr.y (mgr.14152) 115 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:25.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:25 smithi148 bash[21362]: cluster 2024-03-07T03:02:23.183333+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:25.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:25 smithi148 bash[31149]: cluster 2024-03-07T03:02:23.183333+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:25.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:25 smithi195 bash[23221]: cluster 2024-03-07T03:02:23.183333+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:27.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:27 smithi148 bash[21362]: cluster 2024-03-07T03:02:25.183861+0000 mgr.y (mgr.14152) 117 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:27.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:27 smithi148 bash[31149]: cluster 2024-03-07T03:02:25.183861+0000 mgr.y (mgr.14152) 117 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:27.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:27 smithi195 bash[23221]: cluster 2024-03-07T03:02:25.183861+0000 mgr.y (mgr.14152) 117 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:29.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[21362]: cluster 2024-03-07T03:02:27.184298+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:29.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[21362]: audit 2024-03-07T03:02:28.713866+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.148:0/1318788566' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]: dispatch 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[21362]: audit 2024-03-07T03:02:28.714540+0000 mon.a (mon.0) 311 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]: dispatch 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[21362]: audit 2024-03-07T03:02:28.722361+0000 mon.a (mon.0) 312 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]': finished 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[21362]: cluster 2024-03-07T03:02:28.722551+0000 mon.a (mon.0) 313 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[21362]: audit 2024-03-07T03:02:28.722759+0000 mon.a (mon.0) 314 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[31149]: cluster 2024-03-07T03:02:27.184298+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[31149]: audit 2024-03-07T03:02:28.713866+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.148:0/1318788566' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]: dispatch 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[31149]: audit 2024-03-07T03:02:28.714540+0000 mon.a (mon.0) 311 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]: dispatch 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[31149]: audit 2024-03-07T03:02:28.722361+0000 mon.a (mon.0) 312 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]': finished 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[31149]: cluster 2024-03-07T03:02:28.722551+0000 mon.a (mon.0) 313 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-03-07T03:02:29.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:29 smithi148 bash[31149]: audit 2024-03-07T03:02:28.722759+0000 mon.a (mon.0) 314 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:29.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:29 smithi195 bash[23221]: cluster 2024-03-07T03:02:27.184298+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:29.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:29 smithi195 bash[23221]: audit 2024-03-07T03:02:28.713866+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.148:0/1318788566' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]: dispatch 2024-03-07T03:02:29.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:29 smithi195 bash[23221]: audit 2024-03-07T03:02:28.714540+0000 mon.a (mon.0) 311 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]: dispatch 2024-03-07T03:02:29.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:29 smithi195 bash[23221]: audit 2024-03-07T03:02:28.722361+0000 mon.a (mon.0) 312 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8a945bfb-2341-49b2-94cf-6004ef76162f"}]': finished 2024-03-07T03:02:29.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:29 smithi195 bash[23221]: cluster 2024-03-07T03:02:28.722551+0000 mon.a (mon.0) 313 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-03-07T03:02:29.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:29 smithi195 bash[23221]: audit 2024-03-07T03:02:28.722759+0000 mon.a (mon.0) 314 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:30.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:30 smithi148 bash[21362]: audit 2024-03-07T03:02:29.461664+0000 mon.a (mon.0) 315 : audit 0 from='client.? 172.21.15.148:0/4094369211' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:02:30.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:30 smithi148 bash[31149]: audit 2024-03-07T03:02:29.461664+0000 mon.a (mon.0) 315 : audit 0 from='client.? 172.21.15.148:0/4094369211' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:02:30.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:30 smithi195 bash[23221]: audit 2024-03-07T03:02:29.461664+0000 mon.a (mon.0) 315 : audit 0 from='client.? 172.21.15.148:0/4094369211' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:02:31.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:31 smithi148 bash[21362]: cluster 2024-03-07T03:02:29.184547+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v83: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:31.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:31 smithi148 bash[31149]: cluster 2024-03-07T03:02:29.184547+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v83: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:31.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:31 smithi195 bash[23221]: cluster 2024-03-07T03:02:29.184547+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v83: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:33.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:33 smithi148 bash[21362]: cluster 2024-03-07T03:02:31.184802+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:33.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:33 smithi148 bash[31149]: cluster 2024-03-07T03:02:31.184802+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:33.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:33 smithi195 bash[23221]: cluster 2024-03-07T03:02:31.184802+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:35.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:35 smithi148 bash[21362]: cluster 2024-03-07T03:02:33.185189+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v85: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:35.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:35 smithi148 bash[31149]: cluster 2024-03-07T03:02:33.185189+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v85: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:35.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:35 smithi195 bash[23221]: cluster 2024-03-07T03:02:33.185189+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v85: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:37.455 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:37 smithi148 bash[21362]: cluster 2024-03-07T03:02:35.185674+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:37.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:37 smithi148 bash[31149]: cluster 2024-03-07T03:02:35.185674+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:37.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:37 smithi195 bash[23221]: cluster 2024-03-07T03:02:35.185674+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:39.519 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:39 smithi148 bash[21362]: cluster 2024-03-07T03:02:37.186145+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:39.519 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:39 smithi148 bash[31149]: cluster 2024-03-07T03:02:37.186145+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:39.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:39 smithi195 bash[23221]: cluster 2024-03-07T03:02:37.186145+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:41 smithi148 bash[21362]: cluster 2024-03-07T03:02:39.186547+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:41 smithi148 bash[21362]: audit 2024-03-07T03:02:40.583298+0000 mon.a (mon.0) 316 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:02:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:41 smithi148 bash[21362]: audit 2024-03-07T03:02:40.584812+0000 mon.a (mon.0) 317 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:41 smithi148 bash[31149]: cluster 2024-03-07T03:02:39.186547+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:41 smithi148 bash[31149]: audit 2024-03-07T03:02:40.583298+0000 mon.a (mon.0) 316 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:02:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:41 smithi148 bash[31149]: audit 2024-03-07T03:02:40.584812+0000 mon.a (mon.0) 317 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:41 smithi195 bash[23221]: cluster 2024-03-07T03:02:39.186547+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:41 smithi195 bash[23221]: audit 2024-03-07T03:02:40.583298+0000 mon.a (mon.0) 316 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:02:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:41 smithi195 bash[23221]: audit 2024-03-07T03:02:40.584812+0000 mon.a (mon.0) 317 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:42.269 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:42 smithi148 bash[21362]: cephadm 2024-03-07T03:02:40.586221+0000 mgr.y (mgr.14152) 125 : cephadm 1 Deploying daemon osd.1 on smithi148 2024-03-07T03:02:42.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:42 smithi148 bash[31149]: cephadm 2024-03-07T03:02:40.586221+0000 mgr.y (mgr.14152) 125 : cephadm 1 Deploying daemon osd.1 on smithi148 2024-03-07T03:02:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:42 smithi195 bash[23221]: cephadm 2024-03-07T03:02:40.586221+0000 mgr.y (mgr.14152) 125 : cephadm 1 Deploying daemon osd.1 on smithi148 2024-03-07T03:02:43.286 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:43 smithi148 bash[21362]: cluster 2024-03-07T03:02:41.187016+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:43.286 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:43 smithi148 bash[31149]: cluster 2024-03-07T03:02:41.187016+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:43.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:43 smithi195 bash[23221]: cluster 2024-03-07T03:02:41.187016+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:44.300 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:44.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:44.301 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:44.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:44.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:44.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:44.801 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:44.801 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:02:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:02:45.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[21362]: cluster 2024-03-07T03:02:43.187460+0000 mgr.y (mgr.14152) 127 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:45.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[21362]: audit 2024-03-07T03:02:44.525303+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:45.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[21362]: audit 2024-03-07T03:02:44.530126+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:45.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[21362]: audit 2024-03-07T03:02:44.568206+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:45.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[31149]: cluster 2024-03-07T03:02:43.187460+0000 mgr.y (mgr.14152) 127 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:45.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[31149]: audit 2024-03-07T03:02:44.525303+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:45.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[31149]: audit 2024-03-07T03:02:44.530126+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:45.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:45 smithi148 bash[31149]: audit 2024-03-07T03:02:44.568206+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:45.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:45 smithi195 bash[23221]: cluster 2024-03-07T03:02:43.187460+0000 mgr.y (mgr.14152) 127 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:45.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:45 smithi195 bash[23221]: audit 2024-03-07T03:02:44.525303+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:45.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:45 smithi195 bash[23221]: audit 2024-03-07T03:02:44.530126+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:45.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:45 smithi195 bash[23221]: audit 2024-03-07T03:02:44.568206+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:47.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:47 smithi148 bash[21362]: cluster 2024-03-07T03:02:45.187901+0000 mgr.y (mgr.14152) 128 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:47.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:47 smithi148 bash[31149]: cluster 2024-03-07T03:02:45.187901+0000 mgr.y (mgr.14152) 128 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:47.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:47 smithi195 bash[23221]: cluster 2024-03-07T03:02:45.187901+0000 mgr.y (mgr.14152) 128 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:49.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:49 smithi148 bash[21362]: cluster 2024-03-07T03:02:47.188399+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:49 smithi148 bash[31149]: cluster 2024-03-07T03:02:47.188399+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:49.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:49 smithi195 bash[23221]: cluster 2024-03-07T03:02:47.188399+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: cluster 2024-03-07T03:02:49.188895+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: audit 2024-03-07T03:02:49.656525+0000 mon.a (mon.0) 321 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: audit 2024-03-07T03:02:49.662873+0000 mon.a (mon.0) 322 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: audit 2024-03-07T03:02:49.663645+0000 mon.a (mon.0) 323 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: audit 2024-03-07T03:02:49.664292+0000 mon.a (mon.0) 324 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: audit 2024-03-07T03:02:49.670657+0000 mon.a (mon.0) 325 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: audit 2024-03-07T03:02:50.624022+0000 mon.c (mon.2) 7 : audit 1 from='osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[21362]: audit 2024-03-07T03:02:50.624516+0000 mon.a (mon.0) 326 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:02:51.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: cluster 2024-03-07T03:02:49.188895+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:51.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: audit 2024-03-07T03:02:49.656525+0000 mon.a (mon.0) 321 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: audit 2024-03-07T03:02:49.662873+0000 mon.a (mon.0) 322 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: audit 2024-03-07T03:02:49.663645+0000 mon.a (mon.0) 323 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:51.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: audit 2024-03-07T03:02:49.664292+0000 mon.a (mon.0) 324 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:51.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: audit 2024-03-07T03:02:49.670657+0000 mon.a (mon.0) 325 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: audit 2024-03-07T03:02:50.624022+0000 mon.c (mon.2) 7 : audit 1 from='osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:02:51.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:50 smithi148 bash[31149]: audit 2024-03-07T03:02:50.624516+0000 mon.a (mon.0) 326 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:02:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: cluster 2024-03-07T03:02:49.188895+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: audit 2024-03-07T03:02:49.656525+0000 mon.a (mon.0) 321 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: audit 2024-03-07T03:02:49.662873+0000 mon.a (mon.0) 322 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: audit 2024-03-07T03:02:49.663645+0000 mon.a (mon.0) 323 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:02:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: audit 2024-03-07T03:02:49.664292+0000 mon.a (mon.0) 324 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:02:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: audit 2024-03-07T03:02:49.670657+0000 mon.a (mon.0) 325 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: audit 2024-03-07T03:02:50.624022+0000 mon.c (mon.2) 7 : audit 1 from='osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:02:51.075 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:50 smithi195 bash[23221]: audit 2024-03-07T03:02:50.624516+0000 mon.a (mon.0) 326 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:02:52.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[21362]: audit 2024-03-07T03:02:51.181758+0000 mon.a (mon.0) 327 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[21362]: cluster 2024-03-07T03:02:51.181973+0000 mon.a (mon.0) 328 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[21362]: audit 2024-03-07T03:02:51.182212+0000 mon.a (mon.0) 329 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[21362]: audit 2024-03-07T03:02:51.183532+0000 mon.c (mon.2) 8 : audit 1 from='osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[21362]: audit 2024-03-07T03:02:51.184252+0000 mon.a (mon.0) 330 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[31149]: audit 2024-03-07T03:02:51.181758+0000 mon.a (mon.0) 327 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[31149]: cluster 2024-03-07T03:02:51.181973+0000 mon.a (mon.0) 328 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[31149]: audit 2024-03-07T03:02:51.182212+0000 mon.a (mon.0) 329 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[31149]: audit 2024-03-07T03:02:51.183532+0000 mon.c (mon.2) 8 : audit 1 from='osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:52.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:52 smithi148 bash[31149]: audit 2024-03-07T03:02:51.184252+0000 mon.a (mon.0) 330 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:52.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:52 smithi195 bash[23221]: audit 2024-03-07T03:02:51.181758+0000 mon.a (mon.0) 327 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:02:52.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:52 smithi195 bash[23221]: cluster 2024-03-07T03:02:51.181973+0000 mon.a (mon.0) 328 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-03-07T03:02:52.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:52 smithi195 bash[23221]: audit 2024-03-07T03:02:51.182212+0000 mon.a (mon.0) 329 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:52.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:52 smithi195 bash[23221]: audit 2024-03-07T03:02:51.183532+0000 mon.c (mon.2) 8 : audit 1 from='osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:52.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:52 smithi195 bash[23221]: audit 2024-03-07T03:02:51.184252+0000 mon.a (mon.0) 330 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:02:52.754 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 1 on host 'smithi148' 2024-03-07T03:02:53.357 DEBUG:teuthology.orchestra.run.smithi148:osd.1> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.1.service 2024-03-07T03:02:53.360 INFO:tasks.cephadm:Deploying osd.2 on smithi148 with /dev/vg_nvme/lv_2... 2024-03-07T03:02:53.361 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-07T03:02:53.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: cluster 2024-03-07T03:02:51.189332+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:53.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: audit 2024-03-07T03:02:52.181215+0000 mon.a (mon.0) 331 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:02:53.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: cluster 2024-03-07T03:02:52.181316+0000 mon.a (mon.0) 332 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-03-07T03:02:53.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: audit 2024-03-07T03:02:52.181787+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:53.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: audit 2024-03-07T03:02:52.186986+0000 mon.a (mon.0) 334 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: audit 2024-03-07T03:02:52.725888+0000 mon.a (mon.0) 335 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: audit 2024-03-07T03:02:52.735480+0000 mon.a (mon.0) 336 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[21362]: audit 2024-03-07T03:02:52.757952+0000 mon.a (mon.0) 337 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: cluster 2024-03-07T03:02:51.189332+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: audit 2024-03-07T03:02:52.181215+0000 mon.a (mon.0) 331 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: cluster 2024-03-07T03:02:52.181316+0000 mon.a (mon.0) 332 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: audit 2024-03-07T03:02:52.181787+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: audit 2024-03-07T03:02:52.186986+0000 mon.a (mon.0) 334 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: audit 2024-03-07T03:02:52.725888+0000 mon.a (mon.0) 335 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: audit 2024-03-07T03:02:52.735480+0000 mon.a (mon.0) 336 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:53.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:53 smithi148 bash[31149]: audit 2024-03-07T03:02:52.757952+0000 mon.a (mon.0) 337 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: cluster 2024-03-07T03:02:51.189332+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: audit 2024-03-07T03:02:52.181215+0000 mon.a (mon.0) 331 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: cluster 2024-03-07T03:02:52.181316+0000 mon.a (mon.0) 332 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: audit 2024-03-07T03:02:52.181787+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: audit 2024-03-07T03:02:52.186986+0000 mon.a (mon.0) 334 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: audit 2024-03-07T03:02:52.725888+0000 mon.a (mon.0) 335 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: audit 2024-03-07T03:02:52.735480+0000 mon.a (mon.0) 336 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:02:53.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:53 smithi195 bash[23221]: audit 2024-03-07T03:02:52.757952+0000 mon.a (mon.0) 337 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:02:54.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[21362]: cluster 2024-03-07T03:02:51.631777+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[21362]: cluster 2024-03-07T03:02:51.631832+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[21362]: cluster 2024-03-07T03:02:53.190697+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v98: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[21362]: cluster 2024-03-07T03:02:53.191481+0000 mon.a (mon.0) 338 : cluster 1 osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013] boot 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[21362]: cluster 2024-03-07T03:02:53.191610+0000 mon.a (mon.0) 339 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[21362]: audit 2024-03-07T03:02:53.192412+0000 mon.a (mon.0) 340 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[31149]: cluster 2024-03-07T03:02:51.631777+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[31149]: cluster 2024-03-07T03:02:51.631832+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[31149]: cluster 2024-03-07T03:02:53.190697+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v98: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[31149]: cluster 2024-03-07T03:02:53.191481+0000 mon.a (mon.0) 338 : cluster 1 osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013] boot 2024-03-07T03:02:54.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[31149]: cluster 2024-03-07T03:02:53.191610+0000 mon.a (mon.0) 339 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-03-07T03:02:54.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:54 smithi148 bash[31149]: audit 2024-03-07T03:02:53.192412+0000 mon.a (mon.0) 340 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:54.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:54 smithi195 bash[23221]: cluster 2024-03-07T03:02:51.631777+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:02:54.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:54 smithi195 bash[23221]: cluster 2024-03-07T03:02:51.631832+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:02:54.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:54 smithi195 bash[23221]: cluster 2024-03-07T03:02:53.190697+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v98: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-03-07T03:02:54.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:54 smithi195 bash[23221]: cluster 2024-03-07T03:02:53.191481+0000 mon.a (mon.0) 338 : cluster 1 osd.1 [v2:172.21.15.148:6810/3763364013,v1:172.21.15.148:6811/3763364013] boot 2024-03-07T03:02:54.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:54 smithi195 bash[23221]: cluster 2024-03-07T03:02:53.191610+0000 mon.a (mon.0) 339 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-03-07T03:02:54.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:54 smithi195 bash[23221]: audit 2024-03-07T03:02:53.192412+0000 mon.a (mon.0) 340 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:02:55.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:55 smithi148 bash[21362]: cluster 2024-03-07T03:02:54.200279+0000 mon.a (mon.0) 341 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-03-07T03:02:55.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:55 smithi148 bash[31149]: cluster 2024-03-07T03:02:54.200279+0000 mon.a (mon.0) 341 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-03-07T03:02:55.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:55 smithi195 bash[23221]: cluster 2024-03-07T03:02:54.200279+0000 mon.a (mon.0) 341 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-03-07T03:02:56.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:56 smithi148 bash[21362]: cluster 2024-03-07T03:02:55.191144+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v100: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:02:56.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:56 smithi148 bash[31149]: cluster 2024-03-07T03:02:55.191144+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v100: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:02:56.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:56 smithi195 bash[23221]: cluster 2024-03-07T03:02:55.191144+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v100: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:02:57.586 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:02:59.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:02:59 smithi148 bash[31149]: cluster 2024-03-07T03:02:57.191601+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v101: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:02:59.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:02:59 smithi148 bash[21362]: cluster 2024-03-07T03:02:57.191601+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v101: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:02:59.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:02:59 smithi195 bash[23221]: cluster 2024-03-07T03:02:57.191601+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v101: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:01.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:01 smithi148 bash[21362]: cluster 2024-03-07T03:02:59.192080+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:01.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:01 smithi148 bash[31149]: cluster 2024-03-07T03:02:59.192080+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:01.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:01 smithi195 bash[23221]: cluster 2024-03-07T03:02:59.192080+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:01.596 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-07T03:03:01.597 INFO:teuthology.orchestra.run.smithi148.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-07T03:03:01.597 INFO:teuthology.orchestra.run.smithi148.stderr: stderr: 10+0 records in 2024-03-07T03:03:01.597 INFO:teuthology.orchestra.run.smithi148.stderr:10+0 records out 2024-03-07T03:03:01.597 INFO:teuthology.orchestra.run.smithi148.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0139151 s, 754 MB/s 2024-03-07T03:03:01.597 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping successful for: 2024-03-07T03:03:02.324 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi148:vg_nvme/lv_2 2024-03-07T03:03:03.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: cluster 2024-03-07T03:03:01.192401+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v103: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.707125+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.719333+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.721764+0000 mon.a (mon.0) 344 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.723579+0000 mon.a (mon.0) 345 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.735636+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.737987+0000 mon.a (mon.0) 347 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.739825+0000 mon.a (mon.0) 348 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[21362]: audit 2024-03-07T03:03:02.752481+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: cluster 2024-03-07T03:03:01.192401+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v103: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.707125+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.719333+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.721764+0000 mon.a (mon.0) 344 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:03.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.723579+0000 mon.a (mon.0) 345 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:03.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.735636+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.737987+0000 mon.a (mon.0) 347 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:03.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.739825+0000 mon.a (mon.0) 348 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:03.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:03 smithi148 bash[31149]: audit 2024-03-07T03:03:02.752481+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: cluster 2024-03-07T03:03:01.192401+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v103: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.707125+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.719333+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.721764+0000 mon.a (mon.0) 344 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.723579+0000 mon.a (mon.0) 345 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.735636+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.737987+0000 mon.a (mon.0) 347 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.739825+0000 mon.a (mon.0) 348 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:03.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:03 smithi195 bash[23221]: audit 2024-03-07T03:03:02.752481+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:04.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:04 smithi148 bash[21362]: cephadm 2024-03-07T03:03:02.699128+0000 mgr.y (mgr.14152) 137 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:03:04.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:04 smithi148 bash[21362]: cephadm 2024-03-07T03:03:02.724804+0000 mgr.y (mgr.14152) 138 : cephadm 1 Adjusting osd_memory_target on smithi148 to 8120M 2024-03-07T03:03:04.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:04 smithi148 bash[31149]: cephadm 2024-03-07T03:03:02.699128+0000 mgr.y (mgr.14152) 137 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:03:04.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:04 smithi148 bash[31149]: cephadm 2024-03-07T03:03:02.724804+0000 mgr.y (mgr.14152) 138 : cephadm 1 Adjusting osd_memory_target on smithi148 to 8120M 2024-03-07T03:03:04.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:04 smithi195 bash[23221]: cephadm 2024-03-07T03:03:02.699128+0000 mgr.y (mgr.14152) 137 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:03:04.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:04 smithi195 bash[23221]: cephadm 2024-03-07T03:03:02.724804+0000 mgr.y (mgr.14152) 138 : cephadm 1 Adjusting osd_memory_target on smithi148 to 8120M 2024-03-07T03:03:05.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:05 smithi148 bash[21362]: cluster 2024-03-07T03:03:03.192787+0000 mgr.y (mgr.14152) 139 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:05.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:05 smithi148 bash[31149]: cluster 2024-03-07T03:03:03.192787+0000 mgr.y (mgr.14152) 139 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:05.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:05 smithi195 bash[23221]: cluster 2024-03-07T03:03:03.192787+0000 mgr.y (mgr.14152) 139 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:07.010 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:03:07.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:07 smithi148 bash[21362]: cluster 2024-03-07T03:03:05.193276+0000 mgr.y (mgr.14152) 140 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:07.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:07 smithi148 bash[31149]: cluster 2024-03-07T03:03:05.193276+0000 mgr.y (mgr.14152) 140 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:07.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:07 smithi195 bash[23221]: cluster 2024-03-07T03:03:05.193276+0000 mgr.y (mgr.14152) 140 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:09.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:09 smithi148 bash[31149]: cluster 2024-03-07T03:03:07.193778+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:09.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:09 smithi148 bash[21362]: cluster 2024-03-07T03:03:07.193778+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:09.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:09 smithi195 bash[23221]: cluster 2024-03-07T03:03:07.193778+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:10.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:10 smithi148 bash[21362]: audit 2024-03-07T03:03:09.650449+0000 mon.a (mon.0) 350 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:10.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:10 smithi148 bash[21362]: audit 2024-03-07T03:03:09.653643+0000 mon.a (mon.0) 351 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:10.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:10 smithi148 bash[21362]: audit 2024-03-07T03:03:09.654719+0000 mon.a (mon.0) 352 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:10.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:10 smithi148 bash[31149]: audit 2024-03-07T03:03:09.650449+0000 mon.a (mon.0) 350 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:10.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:10 smithi148 bash[31149]: audit 2024-03-07T03:03:09.653643+0000 mon.a (mon.0) 351 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:10.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:10 smithi148 bash[31149]: audit 2024-03-07T03:03:09.654719+0000 mon.a (mon.0) 352 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:10.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:10 smithi195 bash[23221]: audit 2024-03-07T03:03:09.650449+0000 mon.a (mon.0) 350 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:10.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:10 smithi195 bash[23221]: audit 2024-03-07T03:03:09.653643+0000 mon.a (mon.0) 351 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:10.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:10 smithi195 bash[23221]: audit 2024-03-07T03:03:09.654719+0000 mon.a (mon.0) 352 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:11.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:11 smithi148 bash[21362]: cluster 2024-03-07T03:03:09.194200+0000 mgr.y (mgr.14152) 142 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:11.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:11 smithi148 bash[21362]: audit 2024-03-07T03:03:09.643825+0000 mgr.y (mgr.14152) 143 : audit 0 from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:03:11.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:11 smithi148 bash[31149]: cluster 2024-03-07T03:03:09.194200+0000 mgr.y (mgr.14152) 142 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:11.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:11 smithi148 bash[31149]: audit 2024-03-07T03:03:09.643825+0000 mgr.y (mgr.14152) 143 : audit 0 from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:03:11.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:11 smithi195 bash[23221]: cluster 2024-03-07T03:03:09.194200+0000 mgr.y (mgr.14152) 142 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:11.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:11 smithi195 bash[23221]: audit 2024-03-07T03:03:09.643825+0000 mgr.y (mgr.14152) 143 : audit 0 from='client.14256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:03:13.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:13 smithi148 bash[21362]: cluster 2024-03-07T03:03:11.194637+0000 mgr.y (mgr.14152) 144 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:13.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:13 smithi148 bash[31149]: cluster 2024-03-07T03:03:11.194637+0000 mgr.y (mgr.14152) 144 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:13.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:13 smithi195 bash[23221]: cluster 2024-03-07T03:03:11.194637+0000 mgr.y (mgr.14152) 144 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:15.402 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:15 smithi148 bash[21362]: cluster 2024-03-07T03:03:13.195122+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:15.402 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:15 smithi148 bash[31149]: cluster 2024-03-07T03:03:13.195122+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:15.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:15 smithi195 bash[23221]: cluster 2024-03-07T03:03:13.195122+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:17.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:17 smithi148 bash[21362]: cluster 2024-03-07T03:03:15.195610+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:17.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:17 smithi148 bash[31149]: cluster 2024-03-07T03:03:15.195610+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:17.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:17 smithi195 bash[23221]: cluster 2024-03-07T03:03:15.195610+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:18.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[21362]: audit 2024-03-07T03:03:17.599779+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.148:0/3868888214' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]: dispatch 2024-03-07T03:03:18.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[21362]: audit 2024-03-07T03:03:17.600441+0000 mon.a (mon.0) 353 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]: dispatch 2024-03-07T03:03:18.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[21362]: audit 2024-03-07T03:03:17.607616+0000 mon.a (mon.0) 354 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]': finished 2024-03-07T03:03:18.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[21362]: cluster 2024-03-07T03:03:17.607803+0000 mon.a (mon.0) 355 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-03-07T03:03:18.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[21362]: audit 2024-03-07T03:03:17.607947+0000 mon.a (mon.0) 356 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:18.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[31149]: audit 2024-03-07T03:03:17.599779+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.148:0/3868888214' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]: dispatch 2024-03-07T03:03:18.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[31149]: audit 2024-03-07T03:03:17.600441+0000 mon.a (mon.0) 353 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]: dispatch 2024-03-07T03:03:18.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[31149]: audit 2024-03-07T03:03:17.607616+0000 mon.a (mon.0) 354 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]': finished 2024-03-07T03:03:18.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[31149]: cluster 2024-03-07T03:03:17.607803+0000 mon.a (mon.0) 355 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-03-07T03:03:18.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:18 smithi148 bash[31149]: audit 2024-03-07T03:03:17.607947+0000 mon.a (mon.0) 356 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:18 smithi195 bash[23221]: audit 2024-03-07T03:03:17.599779+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.148:0/3868888214' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]: dispatch 2024-03-07T03:03:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:18 smithi195 bash[23221]: audit 2024-03-07T03:03:17.600441+0000 mon.a (mon.0) 353 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]: dispatch 2024-03-07T03:03:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:18 smithi195 bash[23221]: audit 2024-03-07T03:03:17.607616+0000 mon.a (mon.0) 354 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6bb58efa-e4f1-415a-95d5-f4b0df042e11"}]': finished 2024-03-07T03:03:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:18 smithi195 bash[23221]: cluster 2024-03-07T03:03:17.607803+0000 mon.a (mon.0) 355 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-03-07T03:03:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:18 smithi195 bash[23221]: audit 2024-03-07T03:03:17.607947+0000 mon.a (mon.0) 356 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:19.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:19 smithi148 bash[21362]: cluster 2024-03-07T03:03:17.196094+0000 mgr.y (mgr.14152) 147 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:19.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:19 smithi148 bash[21362]: audit 2024-03-07T03:03:18.329920+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.148:0/3061133493' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:19.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:19 smithi148 bash[31149]: cluster 2024-03-07T03:03:17.196094+0000 mgr.y (mgr.14152) 147 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:19.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:19 smithi148 bash[31149]: audit 2024-03-07T03:03:18.329920+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.148:0/3061133493' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:19.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:19 smithi195 bash[23221]: cluster 2024-03-07T03:03:17.196094+0000 mgr.y (mgr.14152) 147 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:19.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:19 smithi195 bash[23221]: audit 2024-03-07T03:03:18.329920+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.148:0/3061133493' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:03:21.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:21 smithi148 bash[21362]: cluster 2024-03-07T03:03:19.196635+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v113: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:21.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:21 smithi148 bash[31149]: cluster 2024-03-07T03:03:19.196635+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v113: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:21.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:21 smithi195 bash[23221]: cluster 2024-03-07T03:03:19.196635+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v113: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:23.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:23 smithi148 bash[21362]: cluster 2024-03-07T03:03:21.197084+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:23.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:23 smithi148 bash[31149]: cluster 2024-03-07T03:03:21.197084+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:23.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:23 smithi195 bash[23221]: cluster 2024-03-07T03:03:21.197084+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:25.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:25 smithi148 bash[21362]: cluster 2024-03-07T03:03:23.197653+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:25.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:25 smithi148 bash[31149]: cluster 2024-03-07T03:03:23.197653+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:25.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:25 smithi195 bash[23221]: cluster 2024-03-07T03:03:23.197653+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:27.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:27 smithi148 bash[21362]: cluster 2024-03-07T03:03:25.198266+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v116: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:27.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:27 smithi148 bash[31149]: cluster 2024-03-07T03:03:25.198266+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v116: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:27.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:27 smithi195 bash[23221]: cluster 2024-03-07T03:03:25.198266+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v116: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:29.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:29 smithi148 bash[21362]: cluster 2024-03-07T03:03:27.198708+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:29.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:29 smithi148 bash[31149]: cluster 2024-03-07T03:03:27.198708+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:29.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:29 smithi195 bash[23221]: cluster 2024-03-07T03:03:27.198708+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:30.501 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:30 smithi148 bash[31149]: audit 2024-03-07T03:03:29.468097+0000 mon.a (mon.0) 357 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:03:30.501 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:30 smithi148 bash[31149]: audit 2024-03-07T03:03:29.469494+0000 mon.a (mon.0) 358 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:30.502 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:30 smithi148 bash[21362]: audit 2024-03-07T03:03:29.468097+0000 mon.a (mon.0) 357 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:03:30.502 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:30 smithi148 bash[21362]: audit 2024-03-07T03:03:29.469494+0000 mon.a (mon.0) 358 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:30.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:30 smithi195 bash[23221]: audit 2024-03-07T03:03:29.468097+0000 mon.a (mon.0) 357 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:03:30.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:30 smithi195 bash[23221]: audit 2024-03-07T03:03:29.469494+0000 mon.a (mon.0) 358 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:31.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:31 smithi148 bash[21362]: cluster 2024-03-07T03:03:29.199141+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:31.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:31 smithi148 bash[21362]: cephadm 2024-03-07T03:03:29.470678+0000 mgr.y (mgr.14152) 154 : cephadm 1 Deploying daemon osd.2 on smithi148 2024-03-07T03:03:31.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:31 smithi148 bash[31149]: cluster 2024-03-07T03:03:29.199141+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:31.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:31 smithi148 bash[31149]: cephadm 2024-03-07T03:03:29.470678+0000 mgr.y (mgr.14152) 154 : cephadm 1 Deploying daemon osd.2 on smithi148 2024-03-07T03:03:31.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:31 smithi195 bash[23221]: cluster 2024-03-07T03:03:29.199141+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:31.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:31 smithi195 bash[23221]: cephadm 2024-03-07T03:03:29.470678+0000 mgr.y (mgr.14152) 154 : cephadm 1 Deploying daemon osd.2 on smithi148 2024-03-07T03:03:33.227 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.227 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.228 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.228 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.228 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:33 smithi148 bash[21362]: cluster 2024-03-07T03:03:31.199448+0000 mgr.y (mgr.14152) 155 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:33.228 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.228 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:33 smithi148 bash[31149]: cluster 2024-03-07T03:03:31.199448+0000 mgr.y (mgr.14152) 155 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:33.514 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.514 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.515 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.515 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.515 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:03:33 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:03:33.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:33 smithi195 bash[23221]: cluster 2024-03-07T03:03:31.199448+0000 mgr.y (mgr.14152) 155 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[21362]: cluster 2024-03-07T03:03:33.199727+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[21362]: audit 2024-03-07T03:03:33.502951+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[21362]: audit 2024-03-07T03:03:33.507911+0000 mon.a (mon.0) 360 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[21362]: audit 2024-03-07T03:03:33.511841+0000 mon.a (mon.0) 361 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[31149]: cluster 2024-03-07T03:03:33.199727+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[31149]: audit 2024-03-07T03:03:33.502951+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[31149]: audit 2024-03-07T03:03:33.507911+0000 mon.a (mon.0) 360 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:34.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:34 smithi148 bash[31149]: audit 2024-03-07T03:03:33.511841+0000 mon.a (mon.0) 361 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:34.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:34 smithi195 bash[23221]: cluster 2024-03-07T03:03:33.199727+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:34.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:34 smithi195 bash[23221]: audit 2024-03-07T03:03:33.502951+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:34.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:34 smithi195 bash[23221]: audit 2024-03-07T03:03:33.507911+0000 mon.a (mon.0) 360 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:34.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:34 smithi195 bash[23221]: audit 2024-03-07T03:03:33.511841+0000 mon.a (mon.0) 361 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:36.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:36 smithi148 bash[21362]: cluster 2024-03-07T03:03:35.200107+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:36.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:36 smithi148 bash[31149]: cluster 2024-03-07T03:03:35.200107+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:36.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:36 smithi195 bash[23221]: cluster 2024-03-07T03:03:35.200107+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:39.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:39 smithi148 bash[21362]: cluster 2024-03-07T03:03:37.200672+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:39.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:39 smithi148 bash[21362]: audit 2024-03-07T03:03:38.694681+0000 mon.a (mon.0) 362 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:39.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:39 smithi148 bash[21362]: audit 2024-03-07T03:03:38.699471+0000 mon.a (mon.0) 363 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:39.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:39 smithi148 bash[31149]: cluster 2024-03-07T03:03:37.200672+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:39.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:39 smithi148 bash[31149]: audit 2024-03-07T03:03:38.694681+0000 mon.a (mon.0) 362 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:39.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:39 smithi148 bash[31149]: audit 2024-03-07T03:03:38.699471+0000 mon.a (mon.0) 363 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:39.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:39 smithi195 bash[23221]: cluster 2024-03-07T03:03:37.200672+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:39.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:39 smithi195 bash[23221]: audit 2024-03-07T03:03:38.694681+0000 mon.a (mon.0) 362 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:39.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:39 smithi195 bash[23221]: audit 2024-03-07T03:03:38.699471+0000 mon.a (mon.0) 363 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:40.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[21362]: audit 2024-03-07T03:03:39.120342+0000 mon.a (mon.0) 364 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[21362]: audit 2024-03-07T03:03:39.121242+0000 mon.a (mon.0) 365 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[21362]: audit 2024-03-07T03:03:39.126616+0000 mon.a (mon.0) 366 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[21362]: audit 2024-03-07T03:03:39.901592+0000 mon.c (mon.2) 11 : audit 1 from='osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[21362]: audit 2024-03-07T03:03:39.901959+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[31149]: audit 2024-03-07T03:03:39.120342+0000 mon.a (mon.0) 364 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[31149]: audit 2024-03-07T03:03:39.121242+0000 mon.a (mon.0) 365 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[31149]: audit 2024-03-07T03:03:39.126616+0000 mon.a (mon.0) 366 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[31149]: audit 2024-03-07T03:03:39.901592+0000 mon.c (mon.2) 11 : audit 1 from='osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:03:40.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:40 smithi148 bash[31149]: audit 2024-03-07T03:03:39.901959+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:03:40.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:40 smithi195 bash[23221]: audit 2024-03-07T03:03:39.120342+0000 mon.a (mon.0) 364 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:40.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:40 smithi195 bash[23221]: audit 2024-03-07T03:03:39.121242+0000 mon.a (mon.0) 365 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:40.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:40 smithi195 bash[23221]: audit 2024-03-07T03:03:39.126616+0000 mon.a (mon.0) 366 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:40.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:40 smithi195 bash[23221]: audit 2024-03-07T03:03:39.901592+0000 mon.c (mon.2) 11 : audit 1 from='osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:03:40.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:40 smithi195 bash[23221]: audit 2024-03-07T03:03:39.901959+0000 mon.a (mon.0) 367 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[21362]: cluster 2024-03-07T03:03:39.201015+0000 mgr.y (mgr.14152) 159 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[21362]: audit 2024-03-07T03:03:40.134099+0000 mon.a (mon.0) 368 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[21362]: cluster 2024-03-07T03:03:40.134189+0000 mon.a (mon.0) 369 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[21362]: audit 2024-03-07T03:03:40.134439+0000 mon.a (mon.0) 370 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[21362]: audit 2024-03-07T03:03:40.135385+0000 mon.c (mon.2) 12 : audit 1 from='osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[21362]: audit 2024-03-07T03:03:40.135797+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[31149]: cluster 2024-03-07T03:03:39.201015+0000 mgr.y (mgr.14152) 159 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[31149]: audit 2024-03-07T03:03:40.134099+0000 mon.a (mon.0) 368 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[31149]: cluster 2024-03-07T03:03:40.134189+0000 mon.a (mon.0) 369 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[31149]: audit 2024-03-07T03:03:40.134439+0000 mon.a (mon.0) 370 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[31149]: audit 2024-03-07T03:03:40.135385+0000 mon.c (mon.2) 12 : audit 1 from='osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:03:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:41 smithi148 bash[31149]: audit 2024-03-07T03:03:40.135797+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:03:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:41 smithi195 bash[23221]: cluster 2024-03-07T03:03:39.201015+0000 mgr.y (mgr.14152) 159 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:41 smithi195 bash[23221]: audit 2024-03-07T03:03:40.134099+0000 mon.a (mon.0) 368 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:03:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:41 smithi195 bash[23221]: cluster 2024-03-07T03:03:40.134189+0000 mon.a (mon.0) 369 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-03-07T03:03:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:41 smithi195 bash[23221]: audit 2024-03-07T03:03:40.134439+0000 mon.a (mon.0) 370 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:41 smithi195 bash[23221]: audit 2024-03-07T03:03:40.135385+0000 mon.c (mon.2) 12 : audit 1 from='osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:03:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:41 smithi195 bash[23221]: audit 2024-03-07T03:03:40.135797+0000 mon.a (mon.0) 371 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:03:41.795 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 2 on host 'smithi148' 2024-03-07T03:03:42.371 DEBUG:teuthology.orchestra.run.smithi148:osd.2> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.2.service 2024-03-07T03:03:42.373 INFO:tasks.cephadm:Deploying osd.3 on smithi148 with /dev/vg_nvme/lv_1... 2024-03-07T03:03:42.373 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[21362]: audit 2024-03-07T03:03:41.137006+0000 mon.a (mon.0) 372 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[21362]: cluster 2024-03-07T03:03:41.137197+0000 mon.a (mon.0) 373 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[21362]: audit 2024-03-07T03:03:41.137604+0000 mon.a (mon.0) 374 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[21362]: audit 2024-03-07T03:03:41.144915+0000 mon.a (mon.0) 375 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[21362]: audit 2024-03-07T03:03:41.788113+0000 mon.a (mon.0) 376 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[21362]: audit 2024-03-07T03:03:41.797994+0000 mon.a (mon.0) 377 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[21362]: audit 2024-03-07T03:03:41.798508+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[31149]: audit 2024-03-07T03:03:41.137006+0000 mon.a (mon.0) 372 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[31149]: cluster 2024-03-07T03:03:41.137197+0000 mon.a (mon.0) 373 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-03-07T03:03:42.425 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[31149]: audit 2024-03-07T03:03:41.137604+0000 mon.a (mon.0) 374 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:42.426 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[31149]: audit 2024-03-07T03:03:41.144915+0000 mon.a (mon.0) 375 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:42.426 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[31149]: audit 2024-03-07T03:03:41.788113+0000 mon.a (mon.0) 376 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:42.426 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[31149]: audit 2024-03-07T03:03:41.797994+0000 mon.a (mon.0) 377 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:42.426 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:42 smithi148 bash[31149]: audit 2024-03-07T03:03:41.798508+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:42.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:42 smithi195 bash[23221]: audit 2024-03-07T03:03:41.137006+0000 mon.a (mon.0) 372 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:03:42.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:42 smithi195 bash[23221]: cluster 2024-03-07T03:03:41.137197+0000 mon.a (mon.0) 373 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-03-07T03:03:42.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:42 smithi195 bash[23221]: audit 2024-03-07T03:03:41.137604+0000 mon.a (mon.0) 374 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:42.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:42 smithi195 bash[23221]: audit 2024-03-07T03:03:41.144915+0000 mon.a (mon.0) 375 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:42 smithi195 bash[23221]: audit 2024-03-07T03:03:41.788113+0000 mon.a (mon.0) 376 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:42 smithi195 bash[23221]: audit 2024-03-07T03:03:41.797994+0000 mon.a (mon.0) 377 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:42 smithi195 bash[23221]: audit 2024-03-07T03:03:41.798508+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[21362]: cluster 2024-03-07T03:03:40.923229+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[21362]: cluster 2024-03-07T03:03:40.923287+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[21362]: cluster 2024-03-07T03:03:41.201429+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v126: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[21362]: audit 2024-03-07T03:03:42.142395+0000 mon.a (mon.0) 379 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[21362]: cluster 2024-03-07T03:03:42.162655+0000 mon.a (mon.0) 380 : cluster 1 osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575] boot 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[21362]: cluster 2024-03-07T03:03:42.162766+0000 mon.a (mon.0) 381 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[21362]: audit 2024-03-07T03:03:42.163341+0000 mon.a (mon.0) 382 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:43.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[31149]: cluster 2024-03-07T03:03:40.923229+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:03:43.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[31149]: cluster 2024-03-07T03:03:40.923287+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:03:43.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[31149]: cluster 2024-03-07T03:03:41.201429+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v126: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:43.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[31149]: audit 2024-03-07T03:03:42.142395+0000 mon.a (mon.0) 379 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:43.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[31149]: cluster 2024-03-07T03:03:42.162655+0000 mon.a (mon.0) 380 : cluster 1 osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575] boot 2024-03-07T03:03:43.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[31149]: cluster 2024-03-07T03:03:42.162766+0000 mon.a (mon.0) 381 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-03-07T03:03:43.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:43 smithi148 bash[31149]: audit 2024-03-07T03:03:42.163341+0000 mon.a (mon.0) 382 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:43.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:43 smithi195 bash[23221]: cluster 2024-03-07T03:03:40.923229+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:03:43.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:43 smithi195 bash[23221]: cluster 2024-03-07T03:03:40.923287+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:03:43.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:43 smithi195 bash[23221]: cluster 2024-03-07T03:03:41.201429+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v126: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-03-07T03:03:43.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:43 smithi195 bash[23221]: audit 2024-03-07T03:03:42.142395+0000 mon.a (mon.0) 379 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:43.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:43 smithi195 bash[23221]: cluster 2024-03-07T03:03:42.162655+0000 mon.a (mon.0) 380 : cluster 1 osd.2 [v2:172.21.15.148:6818/916957575,v1:172.21.15.148:6819/916957575] boot 2024-03-07T03:03:43.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:43 smithi195 bash[23221]: cluster 2024-03-07T03:03:42.162766+0000 mon.a (mon.0) 381 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-03-07T03:03:43.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:43 smithi195 bash[23221]: audit 2024-03-07T03:03:42.163341+0000 mon.a (mon.0) 382 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:03:44.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:44 smithi148 bash[21362]: cluster 2024-03-07T03:03:43.173696+0000 mon.a (mon.0) 383 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-03-07T03:03:44.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:44 smithi148 bash[21362]: audit 2024-03-07T03:03:43.339640+0000 mon.a (mon.0) 384 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:03:44.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:44 smithi148 bash[31149]: cluster 2024-03-07T03:03:43.173696+0000 mon.a (mon.0) 383 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-03-07T03:03:44.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:44 smithi148 bash[31149]: audit 2024-03-07T03:03:43.339640+0000 mon.a (mon.0) 384 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:03:44.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:44 smithi195 bash[23221]: cluster 2024-03-07T03:03:43.173696+0000 mon.a (mon.0) 383 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-03-07T03:03:44.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:44 smithi195 bash[23221]: audit 2024-03-07T03:03:43.339640+0000 mon.a (mon.0) 384 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[21362]: cluster 2024-03-07T03:03:43.201889+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v129: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[21362]: audit 2024-03-07T03:03:44.187936+0000 mon.a (mon.0) 385 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[21362]: cluster 2024-03-07T03:03:44.188067+0000 mon.a (mon.0) 386 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[21362]: audit 2024-03-07T03:03:44.189763+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[31149]: cluster 2024-03-07T03:03:43.201889+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v129: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[31149]: audit 2024-03-07T03:03:44.187936+0000 mon.a (mon.0) 385 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[31149]: cluster 2024-03-07T03:03:44.188067+0000 mon.a (mon.0) 386 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-03-07T03:03:45.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:45 smithi148 bash[31149]: audit 2024-03-07T03:03:44.189763+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:03:45.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:45 smithi195 bash[23221]: cluster 2024-03-07T03:03:43.201889+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v129: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:45.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:45 smithi195 bash[23221]: audit 2024-03-07T03:03:44.187936+0000 mon.a (mon.0) 385 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-03-07T03:03:45.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:45 smithi195 bash[23221]: cluster 2024-03-07T03:03:44.188067+0000 mon.a (mon.0) 386 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-03-07T03:03:45.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:45 smithi195 bash[23221]: audit 2024-03-07T03:03:44.189763+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:03:46.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:46 smithi148 bash[31149]: audit 2024-03-07T03:03:45.194728+0000 mon.a (mon.0) 388 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:03:46.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:46 smithi148 bash[31149]: cluster 2024-03-07T03:03:45.194930+0000 mon.a (mon.0) 389 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-03-07T03:03:46.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:46 smithi148 bash[31149]: cluster 2024-03-07T03:03:45.202746+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v132: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:46.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:46 smithi148 bash[21362]: audit 2024-03-07T03:03:45.194728+0000 mon.a (mon.0) 388 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:03:46.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:46 smithi148 bash[21362]: cluster 2024-03-07T03:03:45.194930+0000 mon.a (mon.0) 389 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-03-07T03:03:46.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:46 smithi148 bash[21362]: cluster 2024-03-07T03:03:45.202746+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v132: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:46.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:46 smithi195 bash[23221]: audit 2024-03-07T03:03:45.194728+0000 mon.a (mon.0) 388 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:03:46.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:46 smithi195 bash[23221]: cluster 2024-03-07T03:03:45.194930+0000 mon.a (mon.0) 389 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-03-07T03:03:46.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:46 smithi195 bash[23221]: cluster 2024-03-07T03:03:45.202746+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v132: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:46.628 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:03:47.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:47 smithi148 bash[21362]: cluster 2024-03-07T03:03:46.215580+0000 mon.a (mon.0) 390 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-03-07T03:03:47.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:47 smithi148 bash[21362]: audit 2024-03-07T03:03:46.897012+0000 mon.a (mon.0) 391 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:47.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:47 smithi148 bash[31149]: cluster 2024-03-07T03:03:46.215580+0000 mon.a (mon.0) 390 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-03-07T03:03:47.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:47 smithi148 bash[31149]: audit 2024-03-07T03:03:46.897012+0000 mon.a (mon.0) 391 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:47.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:47 smithi195 bash[23221]: cluster 2024-03-07T03:03:46.215580+0000 mon.a (mon.0) 390 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-03-07T03:03:47.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:47 smithi195 bash[23221]: audit 2024-03-07T03:03:46.897012+0000 mon.a (mon.0) 391 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:49.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:49 smithi148 bash[21362]: cluster 2024-03-07T03:03:47.203256+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v134: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:49.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:49 smithi148 bash[21362]: cluster 2024-03-07T03:03:48.219299+0000 mon.a (mon.0) 392 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-03-07T03:03:49.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:49 smithi148 bash[21362]: audit 2024-03-07T03:03:48.659895+0000 mon.a (mon.0) 393 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:49.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:49 smithi148 bash[31149]: cluster 2024-03-07T03:03:47.203256+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v134: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:49.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:49 smithi148 bash[31149]: cluster 2024-03-07T03:03:48.219299+0000 mon.a (mon.0) 392 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-03-07T03:03:49.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:49 smithi148 bash[31149]: audit 2024-03-07T03:03:48.659895+0000 mon.a (mon.0) 393 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:49.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:49 smithi195 bash[23221]: cluster 2024-03-07T03:03:47.203256+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v134: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:49.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:49 smithi195 bash[23221]: cluster 2024-03-07T03:03:48.219299+0000 mon.a (mon.0) 392 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-03-07T03:03:49.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:49 smithi195 bash[23221]: audit 2024-03-07T03:03:48.659895+0000 mon.a (mon.0) 393 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:50.511 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-07T03:03:50.511 INFO:teuthology.orchestra.run.smithi148.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-07T03:03:50.511 INFO:teuthology.orchestra.run.smithi148.stderr: stderr: 10+0 records in 2024-03-07T03:03:50.511 INFO:teuthology.orchestra.run.smithi148.stderr:10+0 records out 2024-03-07T03:03:50.511 INFO:teuthology.orchestra.run.smithi148.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0160131 s, 655 MB/s 2024-03-07T03:03:50.511 INFO:teuthology.orchestra.run.smithi148.stderr: stderr: 2024-03-07T03:03:50.511 INFO:teuthology.orchestra.run.smithi148.stderr:--> Zapping successful for: 2024-03-07T03:03:51.269 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi148:vg_nvme/lv_1 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: cluster 2024-03-07T03:03:49.203729+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.546147+0000 mon.a (mon.0) 394 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.547340+0000 mon.a (mon.0) 395 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.547576+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.547921+0000 mon.a (mon.0) 397 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.552928+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.554899+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.555185+0000 mon.a (mon.0) 399 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:49.555463+0000 mon.a (mon.0) 400 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:50.457840+0000 mon.b (mon.1) 9 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:50.485601+0000 mon.a (mon.0) 401 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:51.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:50.488462+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:50.488747+0000 mon.a (mon.0) 403 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[31149]: audit 2024-03-07T03:03:50.488968+0000 mon.a (mon.0) 404 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: cluster 2024-03-07T03:03:49.203729+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.546147+0000 mon.a (mon.0) 394 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.547340+0000 mon.a (mon.0) 395 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.547576+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.547921+0000 mon.a (mon.0) 397 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.552928+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.554899+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.555185+0000 mon.a (mon.0) 399 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:49.555463+0000 mon.a (mon.0) 400 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:50.457840+0000 mon.b (mon.1) 9 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:51.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:50.485601+0000 mon.a (mon.0) 401 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:51.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:50.488462+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:50.488747+0000 mon.a (mon.0) 403 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:51 smithi148 bash[21362]: audit 2024-03-07T03:03:50.488968+0000 mon.a (mon.0) 404 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: cluster 2024-03-07T03:03:49.203729+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.546147+0000 mon.a (mon.0) 394 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.547340+0000 mon.a (mon.0) 395 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.547576+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.547921+0000 mon.a (mon.0) 397 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.552928+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.554899+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.555185+0000 mon.a (mon.0) 399 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:49.555463+0000 mon.a (mon.0) 400 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:50.457840+0000 mon.b (mon.1) 9 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:50.485601+0000 mon.a (mon.0) 401 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:50.488462+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:50.488747+0000 mon.a (mon.0) 403 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:03:51.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:51 smithi195 bash[23221]: audit 2024-03-07T03:03:50.488968+0000 mon.a (mon.0) 404 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:03:52.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:50.488241+0000 mon.c (mon.2) 13 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:52.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: cluster 2024-03-07T03:03:51.204237+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:52.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.362788+0000 mon.c (mon.2) 14 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:52.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: cephadm 2024-03-07T03:03:51.687598+0000 mgr.y (mgr.14152) 166 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:03:52.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.697528+0000 mon.a (mon.0) 405 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.707799+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.710463+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.712469+0000 mon.a (mon.0) 408 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.714379+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: cephadm 2024-03-07T03:03:51.715807+0000 mgr.y (mgr.14152) 167 : cephadm 1 Adjusting osd_memory_target on smithi148 to 5413M 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.724773+0000 mon.a (mon.0) 410 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.727143+0000 mon.a (mon.0) 411 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.728923+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[21362]: audit 2024-03-07T03:03:51.739176+0000 mon.a (mon.0) 413 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:50.488241+0000 mon.c (mon.2) 13 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: cluster 2024-03-07T03:03:51.204237+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:52.801 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.362788+0000 mon.c (mon.2) 14 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: cephadm 2024-03-07T03:03:51.687598+0000 mgr.y (mgr.14152) 166 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.697528+0000 mon.a (mon.0) 405 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.707799+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.710463+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.712469+0000 mon.a (mon.0) 408 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.714379+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: cephadm 2024-03-07T03:03:51.715807+0000 mgr.y (mgr.14152) 167 : cephadm 1 Adjusting osd_memory_target on smithi148 to 5413M 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.724773+0000 mon.a (mon.0) 410 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.727143+0000 mon.a (mon.0) 411 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.728923+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:52 smithi148 bash[31149]: audit 2024-03-07T03:03:51.739176+0000 mon.a (mon.0) 413 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:50.488241+0000 mon.c (mon.2) 13 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: cluster 2024-03-07T03:03:51.204237+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.362788+0000 mon.c (mon.2) 14 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: cephadm 2024-03-07T03:03:51.687598+0000 mgr.y (mgr.14152) 166 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.697528+0000 mon.a (mon.0) 405 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.707799+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.710463+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.712469+0000 mon.a (mon.0) 408 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.714379+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: cephadm 2024-03-07T03:03:51.715807+0000 mgr.y (mgr.14152) 167 : cephadm 1 Adjusting osd_memory_target on smithi148 to 5413M 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.724773+0000 mon.a (mon.0) 410 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.727143+0000 mon.a (mon.0) 411 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.728923+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:03:52.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:52 smithi195 bash[23221]: audit 2024-03-07T03:03:51.739176+0000 mon.a (mon.0) 413 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:03:55.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:55 smithi148 bash[21362]: cluster 2024-03-07T03:03:53.204739+0000 mgr.y (mgr.14152) 168 : cluster 0 pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:55.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:55 smithi148 bash[31149]: cluster 2024-03-07T03:03:53.204739+0000 mgr.y (mgr.14152) 168 : cluster 0 pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:55.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:55 smithi195 bash[23221]: cluster 2024-03-07T03:03:53.204739+0000 mgr.y (mgr.14152) 168 : cluster 0 pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:03:56.007 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:03:57.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:57 smithi148 bash[21362]: cluster 2024-03-07T03:03:55.205330+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-07T03:03:57.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:57 smithi148 bash[31149]: cluster 2024-03-07T03:03:55.205330+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-07T03:03:57.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:57 smithi195 bash[23221]: cluster 2024-03-07T03:03:55.205330+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-07T03:03:59.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[31149]: cluster 2024-03-07T03:03:57.205890+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-07T03:03:59.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[31149]: audit 2024-03-07T03:03:58.607909+0000 mon.a (mon.0) 414 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:59.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[31149]: audit 2024-03-07T03:03:58.611925+0000 mon.a (mon.0) 415 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:59.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[31149]: audit 2024-03-07T03:03:58.613345+0000 mon.a (mon.0) 416 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:59.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[21362]: cluster 2024-03-07T03:03:57.205890+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-07T03:03:59.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[21362]: audit 2024-03-07T03:03:58.607909+0000 mon.a (mon.0) 414 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:59.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[21362]: audit 2024-03-07T03:03:58.611925+0000 mon.a (mon.0) 415 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:59.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:03:59 smithi148 bash[21362]: audit 2024-03-07T03:03:58.613345+0000 mon.a (mon.0) 416 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:03:59.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:59 smithi195 bash[23221]: cluster 2024-03-07T03:03:57.205890+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-07T03:03:59.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:59 smithi195 bash[23221]: audit 2024-03-07T03:03:58.607909+0000 mon.a (mon.0) 414 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:03:59.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:59 smithi195 bash[23221]: audit 2024-03-07T03:03:58.611925+0000 mon.a (mon.0) 415 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:03:59.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:03:59 smithi195 bash[23221]: audit 2024-03-07T03:03:58.613345+0000 mon.a (mon.0) 416 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:00.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:00 smithi148 bash[21362]: audit 2024-03-07T03:03:58.601248+0000 mgr.y (mgr.14152) 171 : audit 0 from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:00.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:00 smithi148 bash[31149]: audit 2024-03-07T03:03:58.601248+0000 mgr.y (mgr.14152) 171 : audit 0 from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:00.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:00 smithi195 bash[23221]: audit 2024-03-07T03:03:58.601248+0000 mgr.y (mgr.14152) 171 : audit 0 from='client.24200 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi148:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:01.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:01 smithi148 bash[31149]: cluster 2024-03-07T03:03:59.206396+0000 mgr.y (mgr.14152) 172 : cluster 0 pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-07T03:04:01.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:01 smithi148 bash[21362]: cluster 2024-03-07T03:03:59.206396+0000 mgr.y (mgr.14152) 172 : cluster 0 pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-07T03:04:01.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:01 smithi195 bash[23221]: cluster 2024-03-07T03:03:59.206396+0000 mgr.y (mgr.14152) 172 : cluster 0 pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-07T03:04:03.473 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:03 smithi148 bash[21362]: cluster 2024-03-07T03:04:01.206929+0000 mgr.y (mgr.14152) 173 : cluster 0 pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:03.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:03 smithi148 bash[31149]: cluster 2024-03-07T03:04:01.206929+0000 mgr.y (mgr.14152) 173 : cluster 0 pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:03.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:03 smithi195 bash[23221]: cluster 2024-03-07T03:04:01.206929+0000 mgr.y (mgr.14152) 173 : cluster 0 pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:05.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:05 smithi148 bash[21362]: cluster 2024-03-07T03:04:03.207475+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:05.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:05 smithi148 bash[31149]: cluster 2024-03-07T03:04:03.207475+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:05.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:05 smithi195 bash[23221]: cluster 2024-03-07T03:04:03.207475+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[21362]: cluster 2024-03-07T03:04:05.208069+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[21362]: audit 2024-03-07T03:04:06.466908+0000 mon.c (mon.2) 15 : audit 1 from='client.? 172.21.15.148:0/1249587770' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]: dispatch 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[21362]: audit 2024-03-07T03:04:06.467382+0000 mon.a (mon.0) 417 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]: dispatch 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[21362]: audit 2024-03-07T03:04:06.477510+0000 mon.a (mon.0) 418 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]': finished 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[21362]: cluster 2024-03-07T03:04:06.477703+0000 mon.a (mon.0) 419 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[21362]: audit 2024-03-07T03:04:06.477873+0000 mon.a (mon.0) 420 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[31149]: cluster 2024-03-07T03:04:05.208069+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[31149]: audit 2024-03-07T03:04:06.466908+0000 mon.c (mon.2) 15 : audit 1 from='client.? 172.21.15.148:0/1249587770' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]: dispatch 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[31149]: audit 2024-03-07T03:04:06.467382+0000 mon.a (mon.0) 417 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]: dispatch 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[31149]: audit 2024-03-07T03:04:06.477510+0000 mon.a (mon.0) 418 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]': finished 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[31149]: cluster 2024-03-07T03:04:06.477703+0000 mon.a (mon.0) 419 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-03-07T03:04:07.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:07 smithi148 bash[31149]: audit 2024-03-07T03:04:06.477873+0000 mon.a (mon.0) 420 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:07 smithi195 bash[23221]: cluster 2024-03-07T03:04:05.208069+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-07T03:04:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:07 smithi195 bash[23221]: audit 2024-03-07T03:04:06.466908+0000 mon.c (mon.2) 15 : audit 1 from='client.? 172.21.15.148:0/1249587770' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]: dispatch 2024-03-07T03:04:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:07 smithi195 bash[23221]: audit 2024-03-07T03:04:06.467382+0000 mon.a (mon.0) 417 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]: dispatch 2024-03-07T03:04:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:07 smithi195 bash[23221]: audit 2024-03-07T03:04:06.477510+0000 mon.a (mon.0) 418 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ed9035a7-5b7b-400e-9293-349d2b59c6a3"}]': finished 2024-03-07T03:04:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:07 smithi195 bash[23221]: cluster 2024-03-07T03:04:06.477703+0000 mon.a (mon.0) 419 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-03-07T03:04:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:07 smithi195 bash[23221]: audit 2024-03-07T03:04:06.477873+0000 mon.a (mon.0) 420 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:08.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:08 smithi148 bash[21362]: audit 2024-03-07T03:04:07.147045+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.148:0/144167587' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:08.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:08 smithi148 bash[31149]: audit 2024-03-07T03:04:07.147045+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.148:0/144167587' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:08.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:08 smithi195 bash[23221]: audit 2024-03-07T03:04:07.147045+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.148:0/144167587' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:09.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:09 smithi148 bash[21362]: cluster 2024-03-07T03:04:07.208449+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:09.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:09 smithi148 bash[31149]: cluster 2024-03-07T03:04:07.208449+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:09.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:09 smithi195 bash[23221]: cluster 2024-03-07T03:04:07.208449+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:11.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:11 smithi148 bash[21362]: cluster 2024-03-07T03:04:09.208904+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:11.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:11 smithi148 bash[31149]: cluster 2024-03-07T03:04:09.208904+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:11.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:11 smithi195 bash[23221]: cluster 2024-03-07T03:04:09.208904+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:13.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:13 smithi148 bash[21362]: cluster 2024-03-07T03:04:11.209418+0000 mgr.y (mgr.14152) 178 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:13.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:13 smithi148 bash[31149]: cluster 2024-03-07T03:04:11.209418+0000 mgr.y (mgr.14152) 178 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:13.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:13 smithi195 bash[23221]: cluster 2024-03-07T03:04:11.209418+0000 mgr.y (mgr.14152) 178 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:15.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:15 smithi148 bash[21362]: cluster 2024-03-07T03:04:13.209946+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:15.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:15 smithi148 bash[31149]: cluster 2024-03-07T03:04:13.209946+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:15.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:15 smithi195 bash[23221]: cluster 2024-03-07T03:04:13.209946+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:17.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:17 smithi148 bash[21362]: cluster 2024-03-07T03:04:15.210396+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:17.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:17 smithi148 bash[31149]: cluster 2024-03-07T03:04:15.210396+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:17.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:17 smithi195 bash[23221]: cluster 2024-03-07T03:04:15.210396+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:19.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:19 smithi148 bash[21362]: cluster 2024-03-07T03:04:17.210912+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:19.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:19 smithi148 bash[21362]: audit 2024-03-07T03:04:18.410571+0000 mon.a (mon.0) 421 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:04:19.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:19 smithi148 bash[21362]: audit 2024-03-07T03:04:18.412455+0000 mon.a (mon.0) 422 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:19.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:19 smithi148 bash[31149]: cluster 2024-03-07T03:04:17.210912+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:19.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:19 smithi148 bash[31149]: audit 2024-03-07T03:04:18.410571+0000 mon.a (mon.0) 421 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:04:19.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:19 smithi148 bash[31149]: audit 2024-03-07T03:04:18.412455+0000 mon.a (mon.0) 422 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:19.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:19 smithi195 bash[23221]: cluster 2024-03-07T03:04:17.210912+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:19.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:19 smithi195 bash[23221]: audit 2024-03-07T03:04:18.410571+0000 mon.a (mon.0) 421 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:04:19.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:19 smithi195 bash[23221]: audit 2024-03-07T03:04:18.412455+0000 mon.a (mon.0) 422 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:20.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:20 smithi148 bash[21362]: cephadm 2024-03-07T03:04:18.414221+0000 mgr.y (mgr.14152) 182 : cephadm 1 Deploying daemon osd.3 on smithi148 2024-03-07T03:04:20.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:20 smithi148 bash[31149]: cephadm 2024-03-07T03:04:18.414221+0000 mgr.y (mgr.14152) 182 : cephadm 1 Deploying daemon osd.3 on smithi148 2024-03-07T03:04:20.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:20 smithi195 bash[23221]: cephadm 2024-03-07T03:04:18.414221+0000 mgr.y (mgr.14152) 182 : cephadm 1 Deploying daemon osd.3 on smithi148 2024-03-07T03:04:21.409 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:21 smithi148 bash[31149]: cluster 2024-03-07T03:04:19.211403+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:21.410 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:21 smithi148 bash[21362]: cluster 2024-03-07T03:04:19.211403+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:21.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:21 smithi195 bash[23221]: cluster 2024-03-07T03:04:19.211403+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:22.414 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.414 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.415 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.415 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.415 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.415 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.799 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.800 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.800 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:22.800 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:04:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[21362]: cluster 2024-03-07T03:04:21.211760+0000 mgr.y (mgr.14152) 184 : cluster 0 pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[21362]: audit 2024-03-07T03:04:22.690611+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[21362]: audit 2024-03-07T03:04:22.696592+0000 mon.a (mon.0) 424 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[21362]: audit 2024-03-07T03:04:22.700827+0000 mon.a (mon.0) 425 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[31149]: cluster 2024-03-07T03:04:21.211760+0000 mgr.y (mgr.14152) 184 : cluster 0 pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[31149]: audit 2024-03-07T03:04:22.690611+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[31149]: audit 2024-03-07T03:04:22.696592+0000 mon.a (mon.0) 424 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:23.445 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:23 smithi148 bash[31149]: audit 2024-03-07T03:04:22.700827+0000 mon.a (mon.0) 425 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:23.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:23 smithi195 bash[23221]: cluster 2024-03-07T03:04:21.211760+0000 mgr.y (mgr.14152) 184 : cluster 0 pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:23.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:23 smithi195 bash[23221]: audit 2024-03-07T03:04:22.690611+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:23.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:23 smithi195 bash[23221]: audit 2024-03-07T03:04:22.696592+0000 mon.a (mon.0) 424 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:23.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:23 smithi195 bash[23221]: audit 2024-03-07T03:04:22.700827+0000 mon.a (mon.0) 425 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:25.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:25 smithi148 bash[21362]: cluster 2024-03-07T03:04:23.212068+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:25.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:25 smithi148 bash[31149]: cluster 2024-03-07T03:04:23.212068+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:25.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:25 smithi195 bash[23221]: cluster 2024-03-07T03:04:23.212068+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:27 smithi148 bash[21362]: cluster 2024-03-07T03:04:25.212565+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:27.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:27 smithi148 bash[31149]: cluster 2024-03-07T03:04:25.212565+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:27.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:27 smithi195 bash[23221]: cluster 2024-03-07T03:04:25.212565+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[21362]: cluster 2024-03-07T03:04:27.212923+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[21362]: audit 2024-03-07T03:04:27.890141+0000 mon.a (mon.0) 426 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[21362]: audit 2024-03-07T03:04:27.896443+0000 mon.a (mon.0) 427 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[21362]: audit 2024-03-07T03:04:27.897294+0000 mon.a (mon.0) 428 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[21362]: audit 2024-03-07T03:04:27.898052+0000 mon.a (mon.0) 429 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[21362]: audit 2024-03-07T03:04:27.904410+0000 mon.a (mon.0) 430 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[31149]: cluster 2024-03-07T03:04:27.212923+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[31149]: audit 2024-03-07T03:04:27.890141+0000 mon.a (mon.0) 426 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:28.963 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[31149]: audit 2024-03-07T03:04:27.896443+0000 mon.a (mon.0) 427 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:28.964 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[31149]: audit 2024-03-07T03:04:27.897294+0000 mon.a (mon.0) 428 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:28.964 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[31149]: audit 2024-03-07T03:04:27.898052+0000 mon.a (mon.0) 429 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:28.964 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:28 smithi148 bash[31149]: audit 2024-03-07T03:04:27.904410+0000 mon.a (mon.0) 430 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:29.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:28 smithi195 bash[23221]: cluster 2024-03-07T03:04:27.212923+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:29.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:28 smithi195 bash[23221]: audit 2024-03-07T03:04:27.890141+0000 mon.a (mon.0) 426 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:29.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:28 smithi195 bash[23221]: audit 2024-03-07T03:04:27.896443+0000 mon.a (mon.0) 427 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:29.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:28 smithi195 bash[23221]: audit 2024-03-07T03:04:27.897294+0000 mon.a (mon.0) 428 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:29.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:28 smithi195 bash[23221]: audit 2024-03-07T03:04:27.898052+0000 mon.a (mon.0) 429 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:29.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:28 smithi195 bash[23221]: audit 2024-03-07T03:04:27.904410+0000 mon.a (mon.0) 430 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:30.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:29 smithi148 bash[21362]: audit 2024-03-07T03:04:28.970882+0000 mon.a (mon.0) 431 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:04:30.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:29 smithi148 bash[31149]: audit 2024-03-07T03:04:28.970882+0000 mon.a (mon.0) 431 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:04:30.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:29 smithi195 bash[23221]: audit 2024-03-07T03:04:28.970882+0000 mon.a (mon.0) 431 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:04:31.112 INFO:teuthology.orchestra.run.smithi148.stdout:Created osd(s) 3 on host 'smithi148' 2024-03-07T03:04:31.164 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[21362]: cluster 2024-03-07T03:04:29.213423+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:31.164 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[21362]: audit 2024-03-07T03:04:29.901167+0000 mon.a (mon.0) 432 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:04:31.164 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[21362]: cluster 2024-03-07T03:04:29.901261+0000 mon.a (mon.0) 433 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-03-07T03:04:31.164 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[21362]: audit 2024-03-07T03:04:29.901419+0000 mon.a (mon.0) 434 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:31.164 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[21362]: audit 2024-03-07T03:04:29.901873+0000 mon.a (mon.0) 435 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:04:31.164 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[31149]: cluster 2024-03-07T03:04:29.213423+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:31.164 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[31149]: audit 2024-03-07T03:04:29.901167+0000 mon.a (mon.0) 432 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:04:31.165 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[31149]: cluster 2024-03-07T03:04:29.901261+0000 mon.a (mon.0) 433 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-03-07T03:04:31.165 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[31149]: audit 2024-03-07T03:04:29.901419+0000 mon.a (mon.0) 434 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:31.165 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:30 smithi148 bash[31149]: audit 2024-03-07T03:04:29.901873+0000 mon.a (mon.0) 435 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:04:31.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:30 smithi195 bash[23221]: cluster 2024-03-07T03:04:29.213423+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:31.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:30 smithi195 bash[23221]: audit 2024-03-07T03:04:29.901167+0000 mon.a (mon.0) 432 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:04:31.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:30 smithi195 bash[23221]: cluster 2024-03-07T03:04:29.901261+0000 mon.a (mon.0) 433 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-03-07T03:04:31.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:30 smithi195 bash[23221]: audit 2024-03-07T03:04:29.901419+0000 mon.a (mon.0) 434 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:31.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:30 smithi195 bash[23221]: audit 2024-03-07T03:04:29.901873+0000 mon.a (mon.0) 435 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:04:31.741 DEBUG:teuthology.orchestra.run.smithi148:osd.3> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.3.service 2024-03-07T03:04:31.744 INFO:tasks.cephadm:Deploying osd.4 on smithi195 with /dev/vg_nvme/lv_4... 2024-03-07T03:04:31.744 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-07T03:04:32.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[21362]: audit 2024-03-07T03:04:30.905292+0000 mon.a (mon.0) 436 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:04:32.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[21362]: cluster 2024-03-07T03:04:30.905537+0000 mon.a (mon.0) 437 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-03-07T03:04:32.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[21362]: audit 2024-03-07T03:04:30.906330+0000 mon.a (mon.0) 438 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[21362]: audit 2024-03-07T03:04:30.914864+0000 mon.a (mon.0) 439 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[21362]: audit 2024-03-07T03:04:31.067035+0000 mon.a (mon.0) 440 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[21362]: audit 2024-03-07T03:04:31.078059+0000 mon.a (mon.0) 441 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[21362]: audit 2024-03-07T03:04:31.078871+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[31149]: audit 2024-03-07T03:04:30.905292+0000 mon.a (mon.0) 436 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[31149]: cluster 2024-03-07T03:04:30.905537+0000 mon.a (mon.0) 437 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[31149]: audit 2024-03-07T03:04:30.906330+0000 mon.a (mon.0) 438 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[31149]: audit 2024-03-07T03:04:30.914864+0000 mon.a (mon.0) 439 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[31149]: audit 2024-03-07T03:04:31.067035+0000 mon.a (mon.0) 440 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[31149]: audit 2024-03-07T03:04:31.078059+0000 mon.a (mon.0) 441 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:32.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:31 smithi148 bash[31149]: audit 2024-03-07T03:04:31.078871+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:32.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:31 smithi195 bash[23221]: audit 2024-03-07T03:04:30.905292+0000 mon.a (mon.0) 436 : audit 1 from='osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]': finished 2024-03-07T03:04:32.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:31 smithi195 bash[23221]: cluster 2024-03-07T03:04:30.905537+0000 mon.a (mon.0) 437 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-03-07T03:04:32.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:31 smithi195 bash[23221]: audit 2024-03-07T03:04:30.906330+0000 mon.a (mon.0) 438 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:32.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:31 smithi195 bash[23221]: audit 2024-03-07T03:04:30.914864+0000 mon.a (mon.0) 439 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:32.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:31 smithi195 bash[23221]: audit 2024-03-07T03:04:31.067035+0000 mon.a (mon.0) 440 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:32.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:31 smithi195 bash[23221]: audit 2024-03-07T03:04:31.078059+0000 mon.a (mon.0) 441 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:32.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:31 smithi195 bash[23221]: audit 2024-03-07T03:04:31.078871+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:04:33.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[31149]: cluster 2024-03-07T03:04:29.968282+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:04:33.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[31149]: cluster 2024-03-07T03:04:29.968363+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:04:33.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[31149]: cluster 2024-03-07T03:04:31.213787+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:33.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[31149]: audit 2024-03-07T03:04:31.917483+0000 mon.a (mon.0) 443 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:33.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[31149]: cluster 2024-03-07T03:04:31.925479+0000 mon.a (mon.0) 444 : cluster 1 osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328] boot 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[31149]: cluster 2024-03-07T03:04:31.925548+0000 mon.a (mon.0) 445 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[31149]: audit 2024-03-07T03:04:31.925952+0000 mon.a (mon.0) 446 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[21362]: cluster 2024-03-07T03:04:29.968282+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[21362]: cluster 2024-03-07T03:04:29.968363+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[21362]: cluster 2024-03-07T03:04:31.213787+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[21362]: audit 2024-03-07T03:04:31.917483+0000 mon.a (mon.0) 443 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[21362]: cluster 2024-03-07T03:04:31.925479+0000 mon.a (mon.0) 444 : cluster 1 osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328] boot 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[21362]: cluster 2024-03-07T03:04:31.925548+0000 mon.a (mon.0) 445 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-03-07T03:04:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:32 smithi148 bash[21362]: audit 2024-03-07T03:04:31.925952+0000 mon.a (mon.0) 446 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:33.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:32 smithi195 bash[23221]: cluster 2024-03-07T03:04:29.968282+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:04:33.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:32 smithi195 bash[23221]: cluster 2024-03-07T03:04:29.968363+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:04:33.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:32 smithi195 bash[23221]: cluster 2024-03-07T03:04:31.213787+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-03-07T03:04:33.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:32 smithi195 bash[23221]: audit 2024-03-07T03:04:31.917483+0000 mon.a (mon.0) 443 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:33.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:32 smithi195 bash[23221]: cluster 2024-03-07T03:04:31.925479+0000 mon.a (mon.0) 444 : cluster 1 osd.3 [v2:172.21.15.148:6826/2267330328,v1:172.21.15.148:6827/2267330328] boot 2024-03-07T03:04:33.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:32 smithi195 bash[23221]: cluster 2024-03-07T03:04:31.925548+0000 mon.a (mon.0) 445 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-03-07T03:04:33.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:32 smithi195 bash[23221]: audit 2024-03-07T03:04:31.925952+0000 mon.a (mon.0) 446 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:04:34.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:33 smithi148 bash[21362]: cluster 2024-03-07T03:04:32.927577+0000 mon.a (mon.0) 447 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-03-07T03:04:34.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:33 smithi148 bash[31149]: cluster 2024-03-07T03:04:32.927577+0000 mon.a (mon.0) 447 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-03-07T03:04:34.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:33 smithi195 bash[23221]: cluster 2024-03-07T03:04:32.927577+0000 mon.a (mon.0) 447 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-03-07T03:04:35.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:35 smithi148 bash[21362]: cluster 2024-03-07T03:04:33.214313+0000 mgr.y (mgr.14152) 190 : cluster 0 pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:35.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:35 smithi148 bash[31149]: cluster 2024-03-07T03:04:33.214313+0000 mgr.y (mgr.14152) 190 : cluster 0 pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:35.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:35 smithi195 bash[23221]: cluster 2024-03-07T03:04:33.214313+0000 mgr.y (mgr.14152) 190 : cluster 0 pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:36.411 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:04:37.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:37 smithi148 bash[21362]: cluster 2024-03-07T03:04:35.214914+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:37.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:37 smithi148 bash[31149]: cluster 2024-03-07T03:04:35.214914+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:37.364 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:37 smithi195 bash[23221]: cluster 2024-03-07T03:04:35.214914+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:39.371 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-07T03:04:39.371 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-07T03:04:39.372 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-03-07T03:04:39.372 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-03-07T03:04:39.372 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0129478 s, 810 MB/s 2024-03-07T03:04:39.372 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-03-07T03:04:39.384 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:39 smithi148 bash[21362]: cluster 2024-03-07T03:04:37.215461+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:39.385 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:39 smithi148 bash[31149]: cluster 2024-03-07T03:04:37.215461+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:39.408 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:39 smithi195 bash[23221]: cluster 2024-03-07T03:04:37.215461+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:39.996 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi195:vg_nvme/lv_4 2024-03-07T03:04:41.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: cluster 2024-03-07T03:04:39.215877+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:41.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: cephadm 2024-03-07T03:04:40.069278+0000 mgr.y (mgr.14152) 194 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:04:41.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.079139+0000 mon.a (mon.0) 448 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.091487+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.094423+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.096287+0000 mon.a (mon.0) 451 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.097962+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.099532+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: cephadm 2024-03-07T03:04:40.100710+0000 mgr.y (mgr.14152) 195 : cephadm 1 Adjusting osd_memory_target on smithi148 to 4060M 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.111149+0000 mon.a (mon.0) 454 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.113786+0000 mon.a (mon.0) 455 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.115350+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[21362]: audit 2024-03-07T03:04:40.129993+0000 mon.a (mon.0) 457 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: cluster 2024-03-07T03:04:39.215877+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: cephadm 2024-03-07T03:04:40.069278+0000 mgr.y (mgr.14152) 194 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:04:41.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.079139+0000 mon.a (mon.0) 448 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.091487+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.094423+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.096287+0000 mon.a (mon.0) 451 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.097962+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.099532+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: cephadm 2024-03-07T03:04:40.100710+0000 mgr.y (mgr.14152) 195 : cephadm 1 Adjusting osd_memory_target on smithi148 to 4060M 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.111149+0000 mon.a (mon.0) 454 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.113786+0000 mon.a (mon.0) 455 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.115350+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:41.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:41 smithi148 bash[31149]: audit 2024-03-07T03:04:40.129993+0000 mon.a (mon.0) 457 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: cluster 2024-03-07T03:04:39.215877+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: cephadm 2024-03-07T03:04:40.069278+0000 mgr.y (mgr.14152) 194 : cephadm 1 Detected new or changed devices on smithi148 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.079139+0000 mon.a (mon.0) 448 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.091487+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.094423+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.096287+0000 mon.a (mon.0) 451 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.097962+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.099532+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:04:41.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: cephadm 2024-03-07T03:04:40.100710+0000 mgr.y (mgr.14152) 195 : cephadm 1 Adjusting osd_memory_target on smithi148 to 4060M 2024-03-07T03:04:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.111149+0000 mon.a (mon.0) 454 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.113786+0000 mon.a (mon.0) 455 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.115350+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:04:41.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:41 smithi195 bash[23221]: audit 2024-03-07T03:04:40.129993+0000 mon.a (mon.0) 457 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:04:43.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:43 smithi148 bash[21362]: cluster 2024-03-07T03:04:41.216405+0000 mgr.y (mgr.14152) 196 : cluster 0 pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:43.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:43 smithi148 bash[31149]: cluster 2024-03-07T03:04:41.216405+0000 mgr.y (mgr.14152) 196 : cluster 0 pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:43.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:43 smithi195 bash[23221]: cluster 2024-03-07T03:04:41.216405+0000 mgr.y (mgr.14152) 196 : cluster 0 pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:44.663 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:04:45.409 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:45 smithi195 bash[23221]: cluster 2024-03-07T03:04:43.216901+0000 mgr.y (mgr.14152) 197 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:45.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:45 smithi148 bash[21362]: cluster 2024-03-07T03:04:43.216901+0000 mgr.y (mgr.14152) 197 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:45.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:45 smithi148 bash[31149]: cluster 2024-03-07T03:04:43.216901+0000 mgr.y (mgr.14152) 197 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:47.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:47 smithi148 bash[21362]: cluster 2024-03-07T03:04:45.217506+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:47.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:47 smithi148 bash[31149]: cluster 2024-03-07T03:04:45.217506+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:47.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:47 smithi195 bash[23221]: cluster 2024-03-07T03:04:45.217506+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:48.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[21362]: audit 2024-03-07T03:04:47.176362+0000 mgr.y (mgr.14152) 199 : audit 0 from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:48.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[21362]: audit 2024-03-07T03:04:47.183822+0000 mon.a (mon.0) 458 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:04:48.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[21362]: audit 2024-03-07T03:04:47.190324+0000 mon.a (mon.0) 459 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:04:48.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[21362]: audit 2024-03-07T03:04:47.192254+0000 mon.a (mon.0) 460 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:48.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[31149]: audit 2024-03-07T03:04:47.176362+0000 mgr.y (mgr.14152) 199 : audit 0 from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:48.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[31149]: audit 2024-03-07T03:04:47.183822+0000 mon.a (mon.0) 458 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:04:48.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[31149]: audit 2024-03-07T03:04:47.190324+0000 mon.a (mon.0) 459 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:04:48.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:48 smithi148 bash[31149]: audit 2024-03-07T03:04:47.192254+0000 mon.a (mon.0) 460 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:48.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:48 smithi195 bash[23221]: audit 2024-03-07T03:04:47.176362+0000 mgr.y (mgr.14152) 199 : audit 0 from='client.24175 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:04:48.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:48 smithi195 bash[23221]: audit 2024-03-07T03:04:47.183822+0000 mon.a (mon.0) 458 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:04:48.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:48 smithi195 bash[23221]: audit 2024-03-07T03:04:47.190324+0000 mon.a (mon.0) 459 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:04:48.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:48 smithi195 bash[23221]: audit 2024-03-07T03:04:47.192254+0000 mon.a (mon.0) 460 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:04:49.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:49 smithi148 bash[21362]: cluster 2024-03-07T03:04:47.217963+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:49 smithi148 bash[31149]: cluster 2024-03-07T03:04:47.217963+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:49.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:49 smithi195 bash[23221]: cluster 2024-03-07T03:04:47.217963+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:51.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:51 smithi148 bash[21362]: cluster 2024-03-07T03:04:49.218498+0000 mgr.y (mgr.14152) 201 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:51.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:51 smithi148 bash[31149]: cluster 2024-03-07T03:04:49.218498+0000 mgr.y (mgr.14152) 201 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:51.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:51 smithi195 bash[23221]: cluster 2024-03-07T03:04:49.218498+0000 mgr.y (mgr.14152) 201 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:53.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:53 smithi148 bash[21362]: cluster 2024-03-07T03:04:51.219017+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:53.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:53 smithi148 bash[31149]: cluster 2024-03-07T03:04:51.219017+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:53.557 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:53 smithi195 bash[23221]: cluster 2024-03-07T03:04:51.219017+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:55.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:55 smithi148 bash[21362]: cluster 2024-03-07T03:04:53.219552+0000 mgr.y (mgr.14152) 203 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:55.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:55 smithi148 bash[31149]: cluster 2024-03-07T03:04:53.219552+0000 mgr.y (mgr.14152) 203 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:55.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:55 smithi195 bash[23221]: cluster 2024-03-07T03:04:53.219552+0000 mgr.y (mgr.14152) 203 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[21362]: audit 2024-03-07T03:04:55.160687+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.195:0/1556087388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]: dispatch 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[21362]: audit 2024-03-07T03:04:55.163695+0000 mon.a (mon.0) 461 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]: dispatch 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[21362]: audit 2024-03-07T03:04:55.172959+0000 mon.a (mon.0) 462 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]': finished 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[21362]: cluster 2024-03-07T03:04:55.173199+0000 mon.a (mon.0) 463 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[21362]: audit 2024-03-07T03:04:55.173442+0000 mon.a (mon.0) 464 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[21362]: audit 2024-03-07T03:04:55.849326+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.195:0/779637233' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[31149]: audit 2024-03-07T03:04:55.160687+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.195:0/1556087388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]: dispatch 2024-03-07T03:04:56.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[31149]: audit 2024-03-07T03:04:55.163695+0000 mon.a (mon.0) 461 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]: dispatch 2024-03-07T03:04:56.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[31149]: audit 2024-03-07T03:04:55.172959+0000 mon.a (mon.0) 462 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]': finished 2024-03-07T03:04:56.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[31149]: cluster 2024-03-07T03:04:55.173199+0000 mon.a (mon.0) 463 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-03-07T03:04:56.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[31149]: audit 2024-03-07T03:04:55.173442+0000 mon.a (mon.0) 464 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:04:56.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:56 smithi148 bash[31149]: audit 2024-03-07T03:04:55.849326+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.195:0/779637233' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:56.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:56 smithi195 bash[23221]: audit 2024-03-07T03:04:55.160687+0000 mon.b (mon.1) 10 : audit 1 from='client.? 172.21.15.195:0/1556087388' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]: dispatch 2024-03-07T03:04:56.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:56 smithi195 bash[23221]: audit 2024-03-07T03:04:55.163695+0000 mon.a (mon.0) 461 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]: dispatch 2024-03-07T03:04:56.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:56 smithi195 bash[23221]: audit 2024-03-07T03:04:55.172959+0000 mon.a (mon.0) 462 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e147b71e-ab98-4dca-ab48-49b4af4fc002"}]': finished 2024-03-07T03:04:56.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:56 smithi195 bash[23221]: cluster 2024-03-07T03:04:55.173199+0000 mon.a (mon.0) 463 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-03-07T03:04:56.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:56 smithi195 bash[23221]: audit 2024-03-07T03:04:55.173442+0000 mon.a (mon.0) 464 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:04:56.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:56 smithi195 bash[23221]: audit 2024-03-07T03:04:55.849326+0000 mon.b (mon.1) 11 : audit 0 from='client.? 172.21.15.195:0/779637233' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:04:57.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:57 smithi148 bash[21362]: cluster 2024-03-07T03:04:55.220056+0000 mgr.y (mgr.14152) 204 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:57.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:57 smithi148 bash[31149]: cluster 2024-03-07T03:04:55.220056+0000 mgr.y (mgr.14152) 204 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:57.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:57 smithi195 bash[23221]: cluster 2024-03-07T03:04:55.220056+0000 mgr.y (mgr.14152) 204 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:59.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:04:59 smithi148 bash[21362]: cluster 2024-03-07T03:04:57.220612+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:59.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:04:59 smithi148 bash[31149]: cluster 2024-03-07T03:04:57.220612+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:04:59.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:04:59 smithi195 bash[23221]: cluster 2024-03-07T03:04:57.220612+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:01.443 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:01 smithi148 bash[21362]: cluster 2024-03-07T03:04:59.221153+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:01.443 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:01 smithi148 bash[31149]: cluster 2024-03-07T03:04:59.221153+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:01.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:01 smithi195 bash[23221]: cluster 2024-03-07T03:04:59.221153+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:03.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:03 smithi148 bash[21362]: cluster 2024-03-07T03:05:01.221629+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:03.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:03 smithi148 bash[31149]: cluster 2024-03-07T03:05:01.221629+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:03.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:03 smithi195 bash[23221]: cluster 2024-03-07T03:05:01.221629+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:05.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:05 smithi148 bash[21362]: cluster 2024-03-07T03:05:03.222144+0000 mgr.y (mgr.14152) 208 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:05.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:05 smithi148 bash[31149]: cluster 2024-03-07T03:05:03.222144+0000 mgr.y (mgr.14152) 208 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:05.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:05 smithi195 bash[23221]: cluster 2024-03-07T03:05:03.222144+0000 mgr.y (mgr.14152) 208 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:07.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:07 smithi148 bash[21362]: cluster 2024-03-07T03:05:05.222777+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:07.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:07 smithi148 bash[21362]: audit 2024-03-07T03:05:07.043584+0000 mon.a (mon.0) 465 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:05:07.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:07 smithi148 bash[21362]: audit 2024-03-07T03:05:07.045564+0000 mon.a (mon.0) 466 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:07 smithi148 bash[31149]: cluster 2024-03-07T03:05:05.222777+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:07 smithi148 bash[31149]: audit 2024-03-07T03:05:07.043584+0000 mon.a (mon.0) 465 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:05:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:07 smithi148 bash[31149]: audit 2024-03-07T03:05:07.045564+0000 mon.a (mon.0) 466 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:07 smithi195 bash[23221]: cluster 2024-03-07T03:05:05.222777+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:07 smithi195 bash[23221]: audit 2024-03-07T03:05:07.043584+0000 mon.a (mon.0) 465 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:05:07.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:07 smithi195 bash[23221]: audit 2024-03-07T03:05:07.045564+0000 mon.a (mon.0) 466 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:08.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:08 smithi195 bash[23221]: cephadm 2024-03-07T03:05:07.047294+0000 mgr.y (mgr.14152) 210 : cephadm 1 Deploying daemon osd.4 on smithi195 2024-03-07T03:05:08.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:08 smithi148 bash[21362]: cephadm 2024-03-07T03:05:07.047294+0000 mgr.y (mgr.14152) 210 : cephadm 1 Deploying daemon osd.4 on smithi195 2024-03-07T03:05:08.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:08 smithi148 bash[31149]: cephadm 2024-03-07T03:05:07.047294+0000 mgr.y (mgr.14152) 210 : cephadm 1 Deploying daemon osd.4 on smithi195 2024-03-07T03:05:09.474 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:09 smithi195 bash[23221]: cluster 2024-03-07T03:05:07.223328+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:09.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:09 smithi148 bash[21362]: cluster 2024-03-07T03:05:07.223328+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:09.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:09 smithi148 bash[31149]: cluster 2024-03-07T03:05:07.223328+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:10.871 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:10 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:10.872 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:05:10 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:11.135 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:11 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:11.136 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:05:11 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:05:11.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[21362]: cluster 2024-03-07T03:05:09.223818+0000 mgr.y (mgr.14152) 212 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:11.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[21362]: audit 2024-03-07T03:05:11.189343+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:11.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[21362]: audit 2024-03-07T03:05:11.199720+0000 mon.a (mon.0) 468 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:11.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[21362]: audit 2024-03-07T03:05:11.200287+0000 mon.a (mon.0) 469 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[31149]: cluster 2024-03-07T03:05:09.223818+0000 mgr.y (mgr.14152) 212 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[31149]: audit 2024-03-07T03:05:11.189343+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[31149]: audit 2024-03-07T03:05:11.199720+0000 mon.a (mon.0) 468 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:11 smithi148 bash[31149]: audit 2024-03-07T03:05:11.200287+0000 mon.a (mon.0) 469 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:11 smithi195 bash[23221]: cluster 2024-03-07T03:05:09.223818+0000 mgr.y (mgr.14152) 212 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:11 smithi195 bash[23221]: audit 2024-03-07T03:05:11.189343+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:11 smithi195 bash[23221]: audit 2024-03-07T03:05:11.199720+0000 mon.a (mon.0) 468 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:11 smithi195 bash[23221]: audit 2024-03-07T03:05:11.200287+0000 mon.a (mon.0) 469 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:13.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:13 smithi148 bash[21362]: cluster 2024-03-07T03:05:11.224325+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:13.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:13 smithi148 bash[31149]: cluster 2024-03-07T03:05:11.224325+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:13.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:13 smithi195 bash[23221]: cluster 2024-03-07T03:05:11.224325+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:14.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:14 smithi195 bash[23221]: cluster 2024-03-07T03:05:13.224836+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:14.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:14 smithi148 bash[21362]: cluster 2024-03-07T03:05:13.224836+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:14.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:14 smithi148 bash[31149]: cluster 2024-03-07T03:05:13.224836+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:16.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:16 smithi148 bash[21362]: cluster 2024-03-07T03:05:15.225410+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:16.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:16 smithi148 bash[31149]: cluster 2024-03-07T03:05:15.225410+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:16.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:16 smithi195 bash[23221]: cluster 2024-03-07T03:05:15.225410+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:17.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:17 smithi195 bash[23221]: audit 2024-03-07T03:05:16.403045+0000 mon.a (mon.0) 470 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:17 smithi195 bash[23221]: audit 2024-03-07T03:05:16.413912+0000 mon.a (mon.0) 471 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:17 smithi195 bash[23221]: audit 2024-03-07T03:05:16.415790+0000 mon.a (mon.0) 472 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:17.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:17 smithi195 bash[23221]: audit 2024-03-07T03:05:16.417638+0000 mon.a (mon.0) 473 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:17.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:17 smithi195 bash[23221]: audit 2024-03-07T03:05:16.430075+0000 mon.a (mon.0) 474 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[21362]: audit 2024-03-07T03:05:16.403045+0000 mon.a (mon.0) 470 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[21362]: audit 2024-03-07T03:05:16.413912+0000 mon.a (mon.0) 471 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[21362]: audit 2024-03-07T03:05:16.415790+0000 mon.a (mon.0) 472 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[21362]: audit 2024-03-07T03:05:16.417638+0000 mon.a (mon.0) 473 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[21362]: audit 2024-03-07T03:05:16.430075+0000 mon.a (mon.0) 474 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[31149]: audit 2024-03-07T03:05:16.403045+0000 mon.a (mon.0) 470 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[31149]: audit 2024-03-07T03:05:16.413912+0000 mon.a (mon.0) 471 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[31149]: audit 2024-03-07T03:05:16.415790+0000 mon.a (mon.0) 472 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[31149]: audit 2024-03-07T03:05:16.417638+0000 mon.a (mon.0) 473 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:17.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:17 smithi148 bash[31149]: audit 2024-03-07T03:05:16.430075+0000 mon.a (mon.0) 474 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:18.798 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:18 smithi148 bash[21362]: cluster 2024-03-07T03:05:17.226055+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:18.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:18 smithi148 bash[21362]: audit 2024-03-07T03:05:17.663760+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:05:18.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:18 smithi148 bash[21362]: audit 2024-03-07T03:05:17.666516+0000 mon.a (mon.0) 475 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:05:18.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:18 smithi148 bash[31149]: cluster 2024-03-07T03:05:17.226055+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:18.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:18 smithi148 bash[31149]: audit 2024-03-07T03:05:17.663760+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:05:18.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:18 smithi148 bash[31149]: audit 2024-03-07T03:05:17.666516+0000 mon.a (mon.0) 475 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:05:18.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:18 smithi195 bash[23221]: cluster 2024-03-07T03:05:17.226055+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:18.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:18 smithi195 bash[23221]: audit 2024-03-07T03:05:17.663760+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:05:18.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:18 smithi195 bash[23221]: audit 2024-03-07T03:05:17.666516+0000 mon.a (mon.0) 475 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:05:19.747 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 4 on host 'smithi195' 2024-03-07T03:05:19.766 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:19 smithi195 bash[23221]: audit 2024-03-07T03:05:18.443305+0000 mon.b (mon.1) 13 : audit 1 from='osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:05:19.766 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:19 smithi195 bash[23221]: audit 2024-03-07T03:05:18.443910+0000 mon.a (mon.0) 476 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:05:19.767 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:19 smithi195 bash[23221]: cluster 2024-03-07T03:05:18.444061+0000 mon.a (mon.0) 477 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-03-07T03:05:19.767 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:19 smithi195 bash[23221]: audit 2024-03-07T03:05:18.444367+0000 mon.a (mon.0) 478 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:19.767 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:19 smithi195 bash[23221]: audit 2024-03-07T03:05:18.446191+0000 mon.a (mon.0) 479 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:05:19.798 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[21362]: audit 2024-03-07T03:05:18.443305+0000 mon.b (mon.1) 13 : audit 1 from='osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:05:19.798 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[21362]: audit 2024-03-07T03:05:18.443910+0000 mon.a (mon.0) 476 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[21362]: cluster 2024-03-07T03:05:18.444061+0000 mon.a (mon.0) 477 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[21362]: audit 2024-03-07T03:05:18.444367+0000 mon.a (mon.0) 478 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[21362]: audit 2024-03-07T03:05:18.446191+0000 mon.a (mon.0) 479 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[31149]: audit 2024-03-07T03:05:18.443305+0000 mon.b (mon.1) 13 : audit 1 from='osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[31149]: audit 2024-03-07T03:05:18.443910+0000 mon.a (mon.0) 476 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[31149]: cluster 2024-03-07T03:05:18.444061+0000 mon.a (mon.0) 477 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[31149]: audit 2024-03-07T03:05:18.444367+0000 mon.a (mon.0) 478 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:19.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:19 smithi148 bash[31149]: audit 2024-03-07T03:05:18.446191+0000 mon.a (mon.0) 479 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:05:20.444 DEBUG:teuthology.orchestra.run.smithi195:osd.4> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.4.service 2024-03-07T03:05:20.446 INFO:tasks.cephadm:Deploying osd.5 on smithi195 with /dev/vg_nvme/lv_3... 2024-03-07T03:05:20.446 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: cluster 2024-03-07T03:05:19.226642+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: audit 2024-03-07T03:05:19.447735+0000 mon.a (mon.0) 480 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: cluster 2024-03-07T03:05:19.447983+0000 mon.a (mon.0) 481 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: audit 2024-03-07T03:05:19.448813+0000 mon.a (mon.0) 482 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: audit 2024-03-07T03:05:19.458436+0000 mon.a (mon.0) 483 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: audit 2024-03-07T03:05:19.701002+0000 mon.a (mon.0) 484 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: audit 2024-03-07T03:05:19.709780+0000 mon.a (mon.0) 485 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[21362]: audit 2024-03-07T03:05:19.723760+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:20.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: cluster 2024-03-07T03:05:19.226642+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: audit 2024-03-07T03:05:19.447735+0000 mon.a (mon.0) 480 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:05:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: cluster 2024-03-07T03:05:19.447983+0000 mon.a (mon.0) 481 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-03-07T03:05:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: audit 2024-03-07T03:05:19.448813+0000 mon.a (mon.0) 482 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: audit 2024-03-07T03:05:19.458436+0000 mon.a (mon.0) 483 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: audit 2024-03-07T03:05:19.701002+0000 mon.a (mon.0) 484 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: audit 2024-03-07T03:05:19.709780+0000 mon.a (mon.0) 485 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:20 smithi148 bash[31149]: audit 2024-03-07T03:05:19.723760+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:20.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: cluster 2024-03-07T03:05:19.226642+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: audit 2024-03-07T03:05:19.447735+0000 mon.a (mon.0) 480 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:05:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: cluster 2024-03-07T03:05:19.447983+0000 mon.a (mon.0) 481 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-03-07T03:05:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: audit 2024-03-07T03:05:19.448813+0000 mon.a (mon.0) 482 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: audit 2024-03-07T03:05:19.458436+0000 mon.a (mon.0) 483 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: audit 2024-03-07T03:05:19.701002+0000 mon.a (mon.0) 484 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: audit 2024-03-07T03:05:19.709780+0000 mon.a (mon.0) 485 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:20 smithi195 bash[23221]: audit 2024-03-07T03:05:19.723760+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:05:21.798 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[21362]: cluster 2024-03-07T03:05:18.638787+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[21362]: cluster 2024-03-07T03:05:18.638895+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[21362]: cluster 2024-03-07T03:05:20.460900+0000 mon.a (mon.0) 487 : cluster 1 osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518] boot 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[21362]: cluster 2024-03-07T03:05:20.460999+0000 mon.a (mon.0) 488 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[21362]: audit 2024-03-07T03:05:20.462058+0000 mon.a (mon.0) 489 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[31149]: cluster 2024-03-07T03:05:18.638787+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[31149]: cluster 2024-03-07T03:05:18.638895+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[31149]: cluster 2024-03-07T03:05:20.460900+0000 mon.a (mon.0) 487 : cluster 1 osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518] boot 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[31149]: cluster 2024-03-07T03:05:20.460999+0000 mon.a (mon.0) 488 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-03-07T03:05:21.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:21 smithi148 bash[31149]: audit 2024-03-07T03:05:20.462058+0000 mon.a (mon.0) 489 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:21.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:21 smithi195 bash[23221]: cluster 2024-03-07T03:05:18.638787+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:05:21.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:21 smithi195 bash[23221]: cluster 2024-03-07T03:05:18.638895+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:05:21.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:21 smithi195 bash[23221]: cluster 2024-03-07T03:05:20.460900+0000 mon.a (mon.0) 487 : cluster 1 osd.4 [v2:172.21.15.195:6800/1363408518,v1:172.21.15.195:6801/1363408518] boot 2024-03-07T03:05:21.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:21 smithi195 bash[23221]: cluster 2024-03-07T03:05:20.460999+0000 mon.a (mon.0) 488 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-03-07T03:05:21.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:21 smithi195 bash[23221]: audit 2024-03-07T03:05:20.462058+0000 mon.a (mon.0) 489 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:05:22.798 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:22 smithi148 bash[21362]: cluster 2024-03-07T03:05:21.227150+0000 mgr.y (mgr.14152) 218 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:22.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:22 smithi148 bash[21362]: cluster 2024-03-07T03:05:21.481600+0000 mon.a (mon.0) 490 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-03-07T03:05:22.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:22 smithi148 bash[31149]: cluster 2024-03-07T03:05:21.227150+0000 mgr.y (mgr.14152) 218 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:22.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:22 smithi148 bash[31149]: cluster 2024-03-07T03:05:21.481600+0000 mon.a (mon.0) 490 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-03-07T03:05:22.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:22 smithi195 bash[23221]: cluster 2024-03-07T03:05:21.227150+0000 mgr.y (mgr.14152) 218 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-03-07T03:05:22.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:22 smithi195 bash[23221]: cluster 2024-03-07T03:05:21.481600+0000 mon.a (mon.0) 490 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-03-07T03:05:23.798 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:23 smithi148 bash[21362]: cluster 2024-03-07T03:05:22.481483+0000 mon.a (mon.0) 491 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-03-07T03:05:23.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:23 smithi148 bash[21362]: cluster 2024-03-07T03:05:23.122452+0000 mon.a (mon.0) 492 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-03-07T03:05:23.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:23 smithi148 bash[31149]: cluster 2024-03-07T03:05:22.481483+0000 mon.a (mon.0) 491 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-03-07T03:05:23.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:23 smithi148 bash[31149]: cluster 2024-03-07T03:05:23.122452+0000 mon.a (mon.0) 492 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-03-07T03:05:23.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:23 smithi195 bash[23221]: cluster 2024-03-07T03:05:22.481483+0000 mon.a (mon.0) 491 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-03-07T03:05:23.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:23 smithi195 bash[23221]: cluster 2024-03-07T03:05:23.122452+0000 mon.a (mon.0) 492 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-03-07T03:05:24.528 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:05:25.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:25 smithi148 bash[21362]: cluster 2024-03-07T03:05:23.227737+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:25.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:25 smithi148 bash[21362]: cluster 2024-03-07T03:05:24.146393+0000 mon.a (mon.0) 493 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-03-07T03:05:25.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:25 smithi148 bash[31149]: cluster 2024-03-07T03:05:23.227737+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:25.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:25 smithi148 bash[31149]: cluster 2024-03-07T03:05:24.146393+0000 mon.a (mon.0) 493 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-03-07T03:05:25.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:25 smithi195 bash[23221]: cluster 2024-03-07T03:05:23.227737+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v194: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:25.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:25 smithi195 bash[23221]: cluster 2024-03-07T03:05:24.146393+0000 mon.a (mon.0) 493 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-03-07T03:05:27.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:27 smithi148 bash[21362]: cluster 2024-03-07T03:05:25.228398+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-07T03:05:27.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:27 smithi148 bash[31149]: cluster 2024-03-07T03:05:25.228398+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-07T03:05:27.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:27 smithi195 bash[23221]: cluster 2024-03-07T03:05:25.228398+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v196: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-07T03:05:28.428 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-07T03:05:28.429 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-07T03:05:28.429 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-03-07T03:05:28.429 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-03-07T03:05:28.429 INFO:teuthology.orchestra.run.smithi195.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0102796 s, 1.0 GB/s 2024-03-07T03:05:28.429 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-03-07T03:05:29.505 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:29 smithi195 bash[23221]: cluster 2024-03-07T03:05:27.229041+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-03-07T03:05:29.506 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi195:vg_nvme/lv_3 2024-03-07T03:05:29.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:29 smithi148 bash[21362]: cluster 2024-03-07T03:05:27.229041+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-03-07T03:05:29.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:29 smithi148 bash[31149]: cluster 2024-03-07T03:05:27.229041+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v197: 1 pgs: 1 active+clean; 577 KiB data, 136 MiB used, 447 GiB / 447 GiB avail; 100 KiB/s, 0 objects/s recovering 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: cluster 2024-03-07T03:05:29.229737+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: cephadm 2024-03-07T03:05:29.671584+0000 mgr.y (mgr.14152) 223 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: audit 2024-03-07T03:05:29.680946+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: audit 2024-03-07T03:05:29.692035+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: audit 2024-03-07T03:05:29.694350+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: cephadm 2024-03-07T03:05:29.695922+0000 mgr.y (mgr.14152) 224 : cephadm 1 Adjusting osd_memory_target on smithi195 to 17265M 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: audit 2024-03-07T03:05:29.706635+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: audit 2024-03-07T03:05:29.708777+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: audit 2024-03-07T03:05:29.710824+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[21362]: audit 2024-03-07T03:05:29.721621+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: cluster 2024-03-07T03:05:29.229737+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-03-07T03:05:31.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: cephadm 2024-03-07T03:05:29.671584+0000 mgr.y (mgr.14152) 223 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: audit 2024-03-07T03:05:29.680946+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: audit 2024-03-07T03:05:29.692035+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: audit 2024-03-07T03:05:29.694350+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: cephadm 2024-03-07T03:05:29.695922+0000 mgr.y (mgr.14152) 224 : cephadm 1 Adjusting osd_memory_target on smithi195 to 17265M 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: audit 2024-03-07T03:05:29.706635+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: audit 2024-03-07T03:05:29.708777+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: audit 2024-03-07T03:05:29.710824+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:30 smithi148 bash[31149]: audit 2024-03-07T03:05:29.721621+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: cluster 2024-03-07T03:05:29.229737+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v198: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 85 KiB/s, 0 objects/s recovering 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: cephadm 2024-03-07T03:05:29.671584+0000 mgr.y (mgr.14152) 223 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: audit 2024-03-07T03:05:29.680946+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: audit 2024-03-07T03:05:29.692035+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: audit 2024-03-07T03:05:29.694350+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: cephadm 2024-03-07T03:05:29.695922+0000 mgr.y (mgr.14152) 224 : cephadm 1 Adjusting osd_memory_target on smithi195 to 17265M 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: audit 2024-03-07T03:05:29.706635+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: audit 2024-03-07T03:05:29.708777+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: audit 2024-03-07T03:05:29.710824+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:05:31.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:30 smithi195 bash[23221]: audit 2024-03-07T03:05:29.721621+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:05:33.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:32 smithi148 bash[21362]: cluster 2024-03-07T03:05:31.230268+0000 mgr.y (mgr.14152) 225 : cluster 0 pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:05:33.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:32 smithi148 bash[31149]: cluster 2024-03-07T03:05:31.230268+0000 mgr.y (mgr.14152) 225 : cluster 0 pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:05:33.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:32 smithi195 bash[23221]: cluster 2024-03-07T03:05:31.230268+0000 mgr.y (mgr.14152) 225 : cluster 0 pgmap v199: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:05:34.185 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:05:35.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:35 smithi195 bash[23221]: cluster 2024-03-07T03:05:33.230840+0000 mgr.y (mgr.14152) 226 : cluster 0 pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:05:35.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:35 smithi148 bash[21362]: cluster 2024-03-07T03:05:33.230840+0000 mgr.y (mgr.14152) 226 : cluster 0 pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:05:35.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:35 smithi148 bash[31149]: cluster 2024-03-07T03:05:33.230840+0000 mgr.y (mgr.14152) 226 : cluster 0 pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:05:37.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[21362]: cluster 2024-03-07T03:05:35.231411+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:05:37.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[21362]: audit 2024-03-07T03:05:36.907057+0000 mon.a (mon.0) 501 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:05:37.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[21362]: audit 2024-03-07T03:05:36.913808+0000 mon.a (mon.0) 502 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:05:37.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[21362]: audit 2024-03-07T03:05:36.915630+0000 mon.a (mon.0) 503 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:37.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[31149]: cluster 2024-03-07T03:05:35.231411+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:05:37.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[31149]: audit 2024-03-07T03:05:36.907057+0000 mon.a (mon.0) 501 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:05:37.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[31149]: audit 2024-03-07T03:05:36.913808+0000 mon.a (mon.0) 502 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:05:37.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:37 smithi148 bash[31149]: audit 2024-03-07T03:05:36.915630+0000 mon.a (mon.0) 503 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:37.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:37 smithi195 bash[23221]: cluster 2024-03-07T03:05:35.231411+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:05:37.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:37 smithi195 bash[23221]: audit 2024-03-07T03:05:36.907057+0000 mon.a (mon.0) 501 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:05:37.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:37 smithi195 bash[23221]: audit 2024-03-07T03:05:36.913808+0000 mon.a (mon.0) 502 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:05:37.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:37 smithi195 bash[23221]: audit 2024-03-07T03:05:36.915630+0000 mon.a (mon.0) 503 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:38.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:38 smithi148 bash[21362]: audit 2024-03-07T03:05:36.900111+0000 mgr.y (mgr.14152) 228 : audit 0 from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:38.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:38 smithi148 bash[31149]: audit 2024-03-07T03:05:36.900111+0000 mgr.y (mgr.14152) 228 : audit 0 from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:38.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:38 smithi195 bash[23221]: audit 2024-03-07T03:05:36.900111+0000 mgr.y (mgr.14152) 228 : audit 0 from='client.24202 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:05:39.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:39 smithi148 bash[21362]: cluster 2024-03-07T03:05:37.231947+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:39.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:39 smithi148 bash[31149]: cluster 2024-03-07T03:05:37.231947+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:39.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:39 smithi195 bash[23221]: cluster 2024-03-07T03:05:37.231947+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:41.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:41 smithi148 bash[21362]: cluster 2024-03-07T03:05:39.232578+0000 mgr.y (mgr.14152) 230 : cluster 0 pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:41.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:41 smithi148 bash[31149]: cluster 2024-03-07T03:05:39.232578+0000 mgr.y (mgr.14152) 230 : cluster 0 pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:41.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:41 smithi195 bash[23221]: cluster 2024-03-07T03:05:39.232578+0000 mgr.y (mgr.14152) 230 : cluster 0 pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:43.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:43 smithi195 bash[23221]: cluster 2024-03-07T03:05:41.233153+0000 mgr.y (mgr.14152) 231 : cluster 0 pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:43.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:43 smithi148 bash[21362]: cluster 2024-03-07T03:05:41.233153+0000 mgr.y (mgr.14152) 231 : cluster 0 pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:43.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:43 smithi148 bash[31149]: cluster 2024-03-07T03:05:41.233153+0000 mgr.y (mgr.14152) 231 : cluster 0 pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:45.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[21362]: cluster 2024-03-07T03:05:43.233693+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:45.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[21362]: audit 2024-03-07T03:05:45.041130+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.195:0/285535626' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]: dispatch 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[21362]: audit 2024-03-07T03:05:45.044063+0000 mon.a (mon.0) 504 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]: dispatch 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[21362]: audit 2024-03-07T03:05:45.053068+0000 mon.a (mon.0) 505 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]': finished 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[21362]: cluster 2024-03-07T03:05:45.053203+0000 mon.a (mon.0) 506 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[21362]: audit 2024-03-07T03:05:45.053466+0000 mon.a (mon.0) 507 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[31149]: cluster 2024-03-07T03:05:43.233693+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[31149]: audit 2024-03-07T03:05:45.041130+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.195:0/285535626' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]: dispatch 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[31149]: audit 2024-03-07T03:05:45.044063+0000 mon.a (mon.0) 504 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]: dispatch 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[31149]: audit 2024-03-07T03:05:45.053068+0000 mon.a (mon.0) 505 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]': finished 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[31149]: cluster 2024-03-07T03:05:45.053203+0000 mon.a (mon.0) 506 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-03-07T03:05:45.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:45 smithi148 bash[31149]: audit 2024-03-07T03:05:45.053466+0000 mon.a (mon.0) 507 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:05:45.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:45 smithi195 bash[23221]: cluster 2024-03-07T03:05:43.233693+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:45.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:45 smithi195 bash[23221]: audit 2024-03-07T03:05:45.041130+0000 mon.b (mon.1) 14 : audit 1 from='client.? 172.21.15.195:0/285535626' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]: dispatch 2024-03-07T03:05:45.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:45 smithi195 bash[23221]: audit 2024-03-07T03:05:45.044063+0000 mon.a (mon.0) 504 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]: dispatch 2024-03-07T03:05:45.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:45 smithi195 bash[23221]: audit 2024-03-07T03:05:45.053068+0000 mon.a (mon.0) 505 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c3b9d5d7-331d-4007-9edf-820635cf80f1"}]': finished 2024-03-07T03:05:45.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:45 smithi195 bash[23221]: cluster 2024-03-07T03:05:45.053203+0000 mon.a (mon.0) 506 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-03-07T03:05:45.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:45 smithi195 bash[23221]: audit 2024-03-07T03:05:45.053466+0000 mon.a (mon.0) 507 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:05:46.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:46 smithi148 bash[21362]: audit 2024-03-07T03:05:45.776209+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.195:0/1167868340' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:05:46.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:46 smithi148 bash[31149]: audit 2024-03-07T03:05:45.776209+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.195:0/1167868340' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:05:46.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:46 smithi195 bash[23221]: audit 2024-03-07T03:05:45.776209+0000 mon.b (mon.1) 15 : audit 0 from='client.? 172.21.15.195:0/1167868340' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:05:47.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:47 smithi148 bash[21362]: cluster 2024-03-07T03:05:45.234224+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:47.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:47 smithi148 bash[31149]: cluster 2024-03-07T03:05:45.234224+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:47.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:47 smithi195 bash[23221]: cluster 2024-03-07T03:05:45.234224+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:49 smithi148 bash[21362]: cluster 2024-03-07T03:05:47.234716+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:49.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:49 smithi148 bash[31149]: cluster 2024-03-07T03:05:47.234716+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:49.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:49 smithi195 bash[23221]: cluster 2024-03-07T03:05:47.234716+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:51.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:51 smithi148 bash[21362]: cluster 2024-03-07T03:05:49.235265+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:51.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:51 smithi148 bash[31149]: cluster 2024-03-07T03:05:49.235265+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:51.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:51 smithi195 bash[23221]: cluster 2024-03-07T03:05:49.235265+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v209: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:53.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:53 smithi148 bash[21362]: cluster 2024-03-07T03:05:51.235924+0000 mgr.y (mgr.14152) 236 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:53.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:53 smithi148 bash[31149]: cluster 2024-03-07T03:05:51.235924+0000 mgr.y (mgr.14152) 236 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:53.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:53 smithi195 bash[23221]: cluster 2024-03-07T03:05:51.235924+0000 mgr.y (mgr.14152) 236 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:55.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:55 smithi148 bash[21362]: cluster 2024-03-07T03:05:53.236576+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:55.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:55 smithi148 bash[31149]: cluster 2024-03-07T03:05:53.236576+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:55.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:55 smithi195 bash[23221]: cluster 2024-03-07T03:05:53.236576+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:57.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:57 smithi148 bash[21362]: cluster 2024-03-07T03:05:55.237122+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:57.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:57 smithi148 bash[21362]: audit 2024-03-07T03:05:56.955539+0000 mon.a (mon.0) 508 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:05:57.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:57 smithi148 bash[21362]: audit 2024-03-07T03:05:56.957287+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:57.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:57 smithi148 bash[31149]: cluster 2024-03-07T03:05:55.237122+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:57.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:57 smithi148 bash[31149]: audit 2024-03-07T03:05:56.955539+0000 mon.a (mon.0) 508 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:05:57.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:57 smithi148 bash[31149]: audit 2024-03-07T03:05:56.957287+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:57.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:57 smithi195 bash[23221]: cluster 2024-03-07T03:05:55.237122+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:57.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:57 smithi195 bash[23221]: audit 2024-03-07T03:05:56.955539+0000 mon.a (mon.0) 508 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:05:57.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:57 smithi195 bash[23221]: audit 2024-03-07T03:05:56.957287+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:05:58.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:58 smithi148 bash[21362]: cephadm 2024-03-07T03:05:56.958850+0000 mgr.y (mgr.14152) 239 : cephadm 1 Deploying daemon osd.5 on smithi195 2024-03-07T03:05:58.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:58 smithi148 bash[31149]: cephadm 2024-03-07T03:05:56.958850+0000 mgr.y (mgr.14152) 239 : cephadm 1 Deploying daemon osd.5 on smithi195 2024-03-07T03:05:58.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:58 smithi195 bash[23221]: cephadm 2024-03-07T03:05:56.958850+0000 mgr.y (mgr.14152) 239 : cephadm 1 Deploying daemon osd.5 on smithi195 2024-03-07T03:05:59.474 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:05:59 smithi195 bash[23221]: cluster 2024-03-07T03:05:57.237758+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:59.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:05:59 smithi148 bash[21362]: cluster 2024-03-07T03:05:57.237758+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:05:59.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:05:59 smithi148 bash[31149]: cluster 2024-03-07T03:05:57.237758+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:00.787 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:06:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:00.788 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:00.788 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:06:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:01.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:01.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:06:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:01.068 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:06:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:01 smithi195 bash[23221]: cluster 2024-03-07T03:05:59.238334+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:01 smithi195 bash[23221]: audit 2024-03-07T03:06:01.073811+0000 mon.a (mon.0) 510 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:01 smithi195 bash[23221]: audit 2024-03-07T03:06:01.082752+0000 mon.a (mon.0) 511 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:01 smithi195 bash[23221]: audit 2024-03-07T03:06:01.096760+0000 mon.a (mon.0) 512 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:01.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[21362]: cluster 2024-03-07T03:05:59.238334+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:01.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[21362]: audit 2024-03-07T03:06:01.073811+0000 mon.a (mon.0) 510 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:01.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[21362]: audit 2024-03-07T03:06:01.082752+0000 mon.a (mon.0) 511 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:01.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[21362]: audit 2024-03-07T03:06:01.096760+0000 mon.a (mon.0) 512 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:01.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[31149]: cluster 2024-03-07T03:05:59.238334+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:01.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[31149]: audit 2024-03-07T03:06:01.073811+0000 mon.a (mon.0) 510 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:01.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[31149]: audit 2024-03-07T03:06:01.082752+0000 mon.a (mon.0) 511 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:01.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:01 smithi148 bash[31149]: audit 2024-03-07T03:06:01.096760+0000 mon.a (mon.0) 512 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:03.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:03 smithi148 bash[21362]: cluster 2024-03-07T03:06:01.238868+0000 mgr.y (mgr.14152) 242 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:03.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:03 smithi148 bash[31149]: cluster 2024-03-07T03:06:01.238868+0000 mgr.y (mgr.14152) 242 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:03.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:03 smithi195 bash[23221]: cluster 2024-03-07T03:06:01.238868+0000 mgr.y (mgr.14152) 242 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:05.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:05 smithi148 bash[21362]: cluster 2024-03-07T03:06:03.239416+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:05.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:05 smithi148 bash[31149]: cluster 2024-03-07T03:06:03.239416+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:05.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:05 smithi195 bash[23221]: cluster 2024-03-07T03:06:03.239416+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:07.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[21362]: cluster 2024-03-07T03:06:05.240085+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:07.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[21362]: audit 2024-03-07T03:06:06.238108+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[21362]: audit 2024-03-07T03:06:06.247414+0000 mon.a (mon.0) 514 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[21362]: audit 2024-03-07T03:06:06.827896+0000 mon.a (mon.0) 515 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[21362]: audit 2024-03-07T03:06:06.829904+0000 mon.a (mon.0) 516 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[21362]: audit 2024-03-07T03:06:06.840386+0000 mon.a (mon.0) 517 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[31149]: cluster 2024-03-07T03:06:05.240085+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[31149]: audit 2024-03-07T03:06:06.238108+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[31149]: audit 2024-03-07T03:06:06.247414+0000 mon.a (mon.0) 514 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[31149]: audit 2024-03-07T03:06:06.827896+0000 mon.a (mon.0) 515 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[31149]: audit 2024-03-07T03:06:06.829904+0000 mon.a (mon.0) 516 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:07.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:07 smithi148 bash[31149]: audit 2024-03-07T03:06:06.840386+0000 mon.a (mon.0) 517 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:07 smithi195 bash[23221]: cluster 2024-03-07T03:06:05.240085+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:07.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:07 smithi195 bash[23221]: audit 2024-03-07T03:06:06.238108+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:07 smithi195 bash[23221]: audit 2024-03-07T03:06:06.247414+0000 mon.a (mon.0) 514 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:07.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:07 smithi195 bash[23221]: audit 2024-03-07T03:06:06.827896+0000 mon.a (mon.0) 515 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:07.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:07 smithi195 bash[23221]: audit 2024-03-07T03:06:06.829904+0000 mon.a (mon.0) 516 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:07.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:07 smithi195 bash[23221]: audit 2024-03-07T03:06:06.840386+0000 mon.a (mon.0) 517 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:08.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:08 smithi148 bash[21362]: audit 2024-03-07T03:06:07.608017+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:06:08.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:08 smithi148 bash[21362]: audit 2024-03-07T03:06:07.610732+0000 mon.a (mon.0) 518 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:06:08.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:08 smithi148 bash[31149]: audit 2024-03-07T03:06:07.608017+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:06:08.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:08 smithi148 bash[31149]: audit 2024-03-07T03:06:07.610732+0000 mon.a (mon.0) 518 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:06:08.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:08 smithi195 bash[23221]: audit 2024-03-07T03:06:07.608017+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:06:08.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:08 smithi195 bash[23221]: audit 2024-03-07T03:06:07.610732+0000 mon.a (mon.0) 518 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-07T03:06:09.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[21362]: cluster 2024-03-07T03:06:07.240631+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:09.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[21362]: audit 2024-03-07T03:06:08.225896+0000 mon.b (mon.1) 17 : audit 1 from='osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:09.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[21362]: audit 2024-03-07T03:06:08.226295+0000 mon.a (mon.0) 519 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[21362]: cluster 2024-03-07T03:06:08.226511+0000 mon.a (mon.0) 520 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[21362]: audit 2024-03-07T03:06:08.226807+0000 mon.a (mon.0) 521 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[21362]: audit 2024-03-07T03:06:08.228740+0000 mon.a (mon.0) 522 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[31149]: cluster 2024-03-07T03:06:07.240631+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[31149]: audit 2024-03-07T03:06:08.225896+0000 mon.b (mon.1) 17 : audit 1 from='osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[31149]: audit 2024-03-07T03:06:08.226295+0000 mon.a (mon.0) 519 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[31149]: cluster 2024-03-07T03:06:08.226511+0000 mon.a (mon.0) 520 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[31149]: audit 2024-03-07T03:06:08.226807+0000 mon.a (mon.0) 521 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:09.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:09 smithi148 bash[31149]: audit 2024-03-07T03:06:08.228740+0000 mon.a (mon.0) 522 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:09.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:09 smithi195 bash[23221]: cluster 2024-03-07T03:06:07.240631+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:09.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:09 smithi195 bash[23221]: audit 2024-03-07T03:06:08.225896+0000 mon.b (mon.1) 17 : audit 1 from='osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:09.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:09 smithi195 bash[23221]: audit 2024-03-07T03:06:08.226295+0000 mon.a (mon.0) 519 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-07T03:06:09.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:09 smithi195 bash[23221]: cluster 2024-03-07T03:06:08.226511+0000 mon.a (mon.0) 520 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-03-07T03:06:09.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:09 smithi195 bash[23221]: audit 2024-03-07T03:06:08.226807+0000 mon.a (mon.0) 521 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:09.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:09 smithi195 bash[23221]: audit 2024-03-07T03:06:08.228740+0000 mon.a (mon.0) 522 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:09.726 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 5 on host 'smithi195' 2024-03-07T03:06:10.414 DEBUG:teuthology.orchestra.run.smithi195:osd.5> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.5.service 2024-03-07T03:06:10.416 INFO:tasks.cephadm:Deploying osd.6 on smithi195 with /dev/vg_nvme/lv_2... 2024-03-07T03:06:10.416 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-07T03:06:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: audit 2024-03-07T03:06:09.227724+0000 mon.a (mon.0) 523 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:06:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: cluster 2024-03-07T03:06:09.227902+0000 mon.a (mon.0) 524 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-03-07T03:06:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: audit 2024-03-07T03:06:09.228655+0000 mon.a (mon.0) 525 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: audit 2024-03-07T03:06:09.237676+0000 mon.a (mon.0) 526 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: cluster 2024-03-07T03:06:09.241116+0000 mgr.y (mgr.14152) 246 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: audit 2024-03-07T03:06:09.685625+0000 mon.a (mon.0) 527 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: audit 2024-03-07T03:06:09.693770+0000 mon.a (mon.0) 528 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[21362]: audit 2024-03-07T03:06:09.702974+0000 mon.a (mon.0) 529 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: audit 2024-03-07T03:06:09.227724+0000 mon.a (mon.0) 523 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: cluster 2024-03-07T03:06:09.227902+0000 mon.a (mon.0) 524 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: audit 2024-03-07T03:06:09.228655+0000 mon.a (mon.0) 525 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: audit 2024-03-07T03:06:09.237676+0000 mon.a (mon.0) 526 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: cluster 2024-03-07T03:06:09.241116+0000 mgr.y (mgr.14152) 246 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: audit 2024-03-07T03:06:09.685625+0000 mon.a (mon.0) 527 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: audit 2024-03-07T03:06:09.693770+0000 mon.a (mon.0) 528 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:10 smithi148 bash[31149]: audit 2024-03-07T03:06:09.702974+0000 mon.a (mon.0) 529 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: audit 2024-03-07T03:06:09.227724+0000 mon.a (mon.0) 523 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: cluster 2024-03-07T03:06:09.227902+0000 mon.a (mon.0) 524 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: audit 2024-03-07T03:06:09.228655+0000 mon.a (mon.0) 525 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: audit 2024-03-07T03:06:09.237676+0000 mon.a (mon.0) 526 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: cluster 2024-03-07T03:06:09.241116+0000 mgr.y (mgr.14152) 246 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: audit 2024-03-07T03:06:09.685625+0000 mon.a (mon.0) 527 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: audit 2024-03-07T03:06:09.693770+0000 mon.a (mon.0) 528 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:10.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:10 smithi195 bash[23221]: audit 2024-03-07T03:06:09.702974+0000 mon.a (mon.0) 529 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:11.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[21362]: cluster 2024-03-07T03:06:08.576032+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:06:11.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[21362]: cluster 2024-03-07T03:06:08.576126+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:06:11.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[21362]: audit 2024-03-07T03:06:10.237280+0000 mon.a (mon.0) 530 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:11.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[21362]: cluster 2024-03-07T03:06:10.247524+0000 mon.a (mon.0) 531 : cluster 1 osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436] boot 2024-03-07T03:06:11.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[21362]: cluster 2024-03-07T03:06:10.247702+0000 mon.a (mon.0) 532 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-03-07T03:06:11.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[21362]: audit 2024-03-07T03:06:10.248496+0000 mon.a (mon.0) 533 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[31149]: cluster 2024-03-07T03:06:08.576032+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:06:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[31149]: cluster 2024-03-07T03:06:08.576126+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:06:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[31149]: audit 2024-03-07T03:06:10.237280+0000 mon.a (mon.0) 530 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[31149]: cluster 2024-03-07T03:06:10.247524+0000 mon.a (mon.0) 531 : cluster 1 osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436] boot 2024-03-07T03:06:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[31149]: cluster 2024-03-07T03:06:10.247702+0000 mon.a (mon.0) 532 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-03-07T03:06:11.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:11 smithi148 bash[31149]: audit 2024-03-07T03:06:10.248496+0000 mon.a (mon.0) 533 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:11.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:11 smithi195 bash[23221]: cluster 2024-03-07T03:06:08.576032+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:06:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:11 smithi195 bash[23221]: cluster 2024-03-07T03:06:08.576126+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:06:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:11 smithi195 bash[23221]: audit 2024-03-07T03:06:10.237280+0000 mon.a (mon.0) 530 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:11 smithi195 bash[23221]: cluster 2024-03-07T03:06:10.247524+0000 mon.a (mon.0) 531 : cluster 1 osd.5 [v2:172.21.15.195:6808/3610217436,v1:172.21.15.195:6809/3610217436] boot 2024-03-07T03:06:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:11 smithi195 bash[23221]: cluster 2024-03-07T03:06:10.247702+0000 mon.a (mon.0) 532 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-03-07T03:06:11.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:11 smithi195 bash[23221]: audit 2024-03-07T03:06:10.248496+0000 mon.a (mon.0) 533 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:06:12.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:12 smithi148 bash[21362]: cluster 2024-03-07T03:06:11.242276+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:12.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:12 smithi148 bash[21362]: cluster 2024-03-07T03:06:11.262592+0000 mon.a (mon.0) 534 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-03-07T03:06:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:12 smithi148 bash[31149]: cluster 2024-03-07T03:06:11.242276+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:12 smithi148 bash[31149]: cluster 2024-03-07T03:06:11.262592+0000 mon.a (mon.0) 534 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-03-07T03:06:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:12 smithi195 bash[23221]: cluster 2024-03-07T03:06:11.242276+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:12 smithi195 bash[23221]: cluster 2024-03-07T03:06:11.262592+0000 mon.a (mon.0) 534 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-03-07T03:06:13.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:13 smithi148 bash[31149]: cluster 2024-03-07T03:06:12.272299+0000 mon.a (mon.0) 535 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-03-07T03:06:13.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:13 smithi148 bash[31149]: cluster 2024-03-07T03:06:13.137808+0000 mon.a (mon.0) 536 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-03-07T03:06:13.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:13 smithi148 bash[21362]: cluster 2024-03-07T03:06:12.272299+0000 mon.a (mon.0) 535 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-03-07T03:06:13.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:13 smithi148 bash[21362]: cluster 2024-03-07T03:06:13.137808+0000 mon.a (mon.0) 536 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-03-07T03:06:13.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:13 smithi195 bash[23221]: cluster 2024-03-07T03:06:12.272299+0000 mon.a (mon.0) 535 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-03-07T03:06:13.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:13 smithi195 bash[23221]: cluster 2024-03-07T03:06:13.137808+0000 mon.a (mon.0) 536 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-03-07T03:06:14.535 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:06:15.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:15 smithi195 bash[23221]: cluster 2024-03-07T03:06:13.242878+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v227: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:15.536 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:15 smithi195 bash[23221]: cluster 2024-03-07T03:06:14.140618+0000 mon.a (mon.0) 537 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-03-07T03:06:15.537 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:15 smithi195 bash[23221]: cluster 2024-03-07T03:06:14.140980+0000 mon.a (mon.0) 538 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:06:15.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:15 smithi148 bash[21362]: cluster 2024-03-07T03:06:13.242878+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v227: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:15.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:15 smithi148 bash[21362]: cluster 2024-03-07T03:06:14.140618+0000 mon.a (mon.0) 537 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-03-07T03:06:15.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:15 smithi148 bash[21362]: cluster 2024-03-07T03:06:14.140980+0000 mon.a (mon.0) 538 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:06:15.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:15 smithi148 bash[31149]: cluster 2024-03-07T03:06:13.242878+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v227: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:15.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:15 smithi148 bash[31149]: cluster 2024-03-07T03:06:14.140618+0000 mon.a (mon.0) 537 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-03-07T03:06:15.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:15 smithi148 bash[31149]: cluster 2024-03-07T03:06:14.140980+0000 mon.a (mon.0) 538 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:06:17.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:17 smithi148 bash[21362]: cluster 2024-03-07T03:06:15.243510+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:17.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:17 smithi148 bash[31149]: cluster 2024-03-07T03:06:15.243510+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:17.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:17 smithi195 bash[23221]: cluster 2024-03-07T03:06:15.243510+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v229: 1 pgs: 1 remapped+peering; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:18.694 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-07T03:06:18.695 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-07T03:06:18.695 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-03-07T03:06:18.695 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-03-07T03:06:18.695 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114053 s, 919 MB/s 2024-03-07T03:06:18.695 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-03-07T03:06:19.330 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:19 smithi195 bash[23221]: cluster 2024-03-07T03:06:17.244103+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:19.330 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:19 smithi195 bash[23221]: cluster 2024-03-07T03:06:18.140259+0000 mon.a (mon.0) 539 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-07T03:06:19.330 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:19 smithi195 bash[23221]: cluster 2024-03-07T03:06:18.140313+0000 mon.a (mon.0) 540 : cluster 1 Cluster is now healthy 2024-03-07T03:06:19.483 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi195:vg_nvme/lv_2 2024-03-07T03:06:19.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:19 smithi148 bash[21362]: cluster 2024-03-07T03:06:17.244103+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:19.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:19 smithi148 bash[21362]: cluster 2024-03-07T03:06:18.140259+0000 mon.a (mon.0) 539 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-07T03:06:19.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:19 smithi148 bash[21362]: cluster 2024-03-07T03:06:18.140313+0000 mon.a (mon.0) 540 : cluster 1 Cluster is now healthy 2024-03-07T03:06:19.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:19 smithi148 bash[31149]: cluster 2024-03-07T03:06:17.244103+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v230: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:19.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:19 smithi148 bash[31149]: cluster 2024-03-07T03:06:18.140259+0000 mon.a (mon.0) 539 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-07T03:06:19.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:19 smithi148 bash[31149]: cluster 2024-03-07T03:06:18.140313+0000 mon.a (mon.0) 540 : cluster 1 Cluster is now healthy 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: cluster 2024-03-07T03:06:19.244635+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: cephadm 2024-03-07T03:06:19.931099+0000 mgr.y (mgr.14152) 252 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.940603+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.952183+0000 mon.a (mon.0) 542 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.954632+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.956450+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: cephadm 2024-03-07T03:06:19.958119+0000 mgr.y (mgr.14152) 253 : cephadm 1 Adjusting osd_memory_target on smithi195 to 8632M 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.967019+0000 mon.a (mon.0) 545 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.969194+0000 mon.a (mon.0) 546 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:21.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.971085+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[21362]: audit 2024-03-07T03:06:19.981570+0000 mon.a (mon.0) 548 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: cluster 2024-03-07T03:06:19.244635+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: cephadm 2024-03-07T03:06:19.931099+0000 mgr.y (mgr.14152) 252 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.940603+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.952183+0000 mon.a (mon.0) 542 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.954632+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.956450+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: cephadm 2024-03-07T03:06:19.958119+0000 mgr.y (mgr.14152) 253 : cephadm 1 Adjusting osd_memory_target on smithi195 to 8632M 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.967019+0000 mon.a (mon.0) 545 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.969194+0000 mon.a (mon.0) 546 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.971085+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:21.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:20 smithi148 bash[31149]: audit 2024-03-07T03:06:19.981570+0000 mon.a (mon.0) 548 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: cluster 2024-03-07T03:06:19.244635+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v231: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: cephadm 2024-03-07T03:06:19.931099+0000 mgr.y (mgr.14152) 252 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.940603+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.952183+0000 mon.a (mon.0) 542 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.954632+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.956450+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: cephadm 2024-03-07T03:06:19.958119+0000 mgr.y (mgr.14152) 253 : cephadm 1 Adjusting osd_memory_target on smithi195 to 8632M 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.967019+0000 mon.a (mon.0) 545 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.969194+0000 mon.a (mon.0) 546 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.971085+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:20 smithi195 bash[23221]: audit 2024-03-07T03:06:19.981570+0000 mon.a (mon.0) 548 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:23.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:22 smithi148 bash[21362]: cluster 2024-03-07T03:06:21.245210+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:06:23.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:22 smithi148 bash[31149]: cluster 2024-03-07T03:06:21.245210+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:06:23.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:22 smithi195 bash[23221]: cluster 2024-03-07T03:06:21.245210+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v232: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:06:24.167 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:06:25.297 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:25 smithi195 bash[23221]: cluster 2024-03-07T03:06:23.245746+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:06:25.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:25 smithi148 bash[21362]: cluster 2024-03-07T03:06:23.245746+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:06:25.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:25 smithi148 bash[31149]: cluster 2024-03-07T03:06:23.245746+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:06:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[21362]: cluster 2024-03-07T03:06:25.246375+0000 mgr.y (mgr.14152) 256 : cluster 0 pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:06:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[21362]: audit 2024-03-07T03:06:26.984242+0000 mon.a (mon.0) 549 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:06:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[21362]: audit 2024-03-07T03:06:26.990661+0000 mon.a (mon.0) 550 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:06:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[21362]: audit 2024-03-07T03:06:26.992318+0000 mon.a (mon.0) 551 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[31149]: cluster 2024-03-07T03:06:25.246375+0000 mgr.y (mgr.14152) 256 : cluster 0 pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:06:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[31149]: audit 2024-03-07T03:06:26.984242+0000 mon.a (mon.0) 549 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:06:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[31149]: audit 2024-03-07T03:06:26.990661+0000 mon.a (mon.0) 550 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:06:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:27 smithi148 bash[31149]: audit 2024-03-07T03:06:26.992318+0000 mon.a (mon.0) 551 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:27.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:27 smithi195 bash[23221]: cluster 2024-03-07T03:06:25.246375+0000 mgr.y (mgr.14152) 256 : cluster 0 pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:06:27.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:27 smithi195 bash[23221]: audit 2024-03-07T03:06:26.984242+0000 mon.a (mon.0) 549 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:06:27.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:27 smithi195 bash[23221]: audit 2024-03-07T03:06:26.990661+0000 mon.a (mon.0) 550 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:06:27.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:27 smithi195 bash[23221]: audit 2024-03-07T03:06:26.992318+0000 mon.a (mon.0) 551 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:28.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:28 smithi148 bash[21362]: audit 2024-03-07T03:06:26.977246+0000 mgr.y (mgr.14152) 257 : audit 0 from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:28.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:28 smithi148 bash[31149]: audit 2024-03-07T03:06:26.977246+0000 mgr.y (mgr.14152) 257 : audit 0 from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:28.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:28 smithi195 bash[23221]: audit 2024-03-07T03:06:26.977246+0000 mgr.y (mgr.14152) 257 : audit 0 from='client.24229 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:06:29.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:29 smithi148 bash[21362]: cluster 2024-03-07T03:06:27.246997+0000 mgr.y (mgr.14152) 258 : cluster 0 pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:06:29.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:29 smithi148 bash[31149]: cluster 2024-03-07T03:06:27.246997+0000 mgr.y (mgr.14152) 258 : cluster 0 pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:06:29.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:29 smithi195 bash[23221]: cluster 2024-03-07T03:06:27.246997+0000 mgr.y (mgr.14152) 258 : cluster 0 pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:06:31.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:31 smithi148 bash[21362]: cluster 2024-03-07T03:06:29.247571+0000 mgr.y (mgr.14152) 259 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:31.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:31 smithi148 bash[31149]: cluster 2024-03-07T03:06:29.247571+0000 mgr.y (mgr.14152) 259 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:31.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:31 smithi195 bash[23221]: cluster 2024-03-07T03:06:29.247571+0000 mgr.y (mgr.14152) 259 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:33.330 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:33 smithi195 bash[23221]: cluster 2024-03-07T03:06:31.248181+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:33.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:33 smithi148 bash[21362]: cluster 2024-03-07T03:06:31.248181+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:33.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:33 smithi148 bash[31149]: cluster 2024-03-07T03:06:31.248181+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:35.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:35 smithi148 bash[21362]: cluster 2024-03-07T03:06:33.248684+0000 mgr.y (mgr.14152) 261 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:35.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:35 smithi148 bash[31149]: cluster 2024-03-07T03:06:33.248684+0000 mgr.y (mgr.14152) 261 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:35.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:35 smithi195 bash[23221]: cluster 2024-03-07T03:06:33.248684+0000 mgr.y (mgr.14152) 261 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:36.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[21362]: audit 2024-03-07T03:06:35.249123+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.195:0/555626629' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]: dispatch 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[21362]: audit 2024-03-07T03:06:35.251744+0000 mon.a (mon.0) 552 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]: dispatch 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[21362]: audit 2024-03-07T03:06:35.260673+0000 mon.a (mon.0) 553 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]': finished 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[21362]: cluster 2024-03-07T03:06:35.260884+0000 mon.a (mon.0) 554 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[21362]: audit 2024-03-07T03:06:35.261179+0000 mon.a (mon.0) 555 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[21362]: audit 2024-03-07T03:06:35.976923+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.195:0/4118793064' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[31149]: audit 2024-03-07T03:06:35.249123+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.195:0/555626629' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]: dispatch 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[31149]: audit 2024-03-07T03:06:35.251744+0000 mon.a (mon.0) 552 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]: dispatch 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[31149]: audit 2024-03-07T03:06:35.260673+0000 mon.a (mon.0) 553 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]': finished 2024-03-07T03:06:36.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[31149]: cluster 2024-03-07T03:06:35.260884+0000 mon.a (mon.0) 554 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-03-07T03:06:36.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[31149]: audit 2024-03-07T03:06:35.261179+0000 mon.a (mon.0) 555 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:36.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:36 smithi148 bash[31149]: audit 2024-03-07T03:06:35.976923+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.195:0/4118793064' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:06:36.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:36 smithi195 bash[23221]: audit 2024-03-07T03:06:35.249123+0000 mon.b (mon.1) 18 : audit 1 from='client.? 172.21.15.195:0/555626629' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]: dispatch 2024-03-07T03:06:36.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:36 smithi195 bash[23221]: audit 2024-03-07T03:06:35.251744+0000 mon.a (mon.0) 552 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]: dispatch 2024-03-07T03:06:36.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:36 smithi195 bash[23221]: audit 2024-03-07T03:06:35.260673+0000 mon.a (mon.0) 553 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "3676cb41-e06f-41d8-b9cc-0271403b23a8"}]': finished 2024-03-07T03:06:36.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:36 smithi195 bash[23221]: cluster 2024-03-07T03:06:35.260884+0000 mon.a (mon.0) 554 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-03-07T03:06:36.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:36 smithi195 bash[23221]: audit 2024-03-07T03:06:35.261179+0000 mon.a (mon.0) 555 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:36.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:36 smithi195 bash[23221]: audit 2024-03-07T03:06:35.976923+0000 mon.b (mon.1) 19 : audit 0 from='client.? 172.21.15.195:0/4118793064' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:06:37.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:37 smithi148 bash[21362]: cluster 2024-03-07T03:06:35.249225+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:37.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:37 smithi148 bash[31149]: cluster 2024-03-07T03:06:35.249225+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:37.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:37 smithi195 bash[23221]: cluster 2024-03-07T03:06:35.249225+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:39.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:39 smithi148 bash[21362]: cluster 2024-03-07T03:06:37.249786+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:39.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:39 smithi148 bash[31149]: cluster 2024-03-07T03:06:37.249786+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:39.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:39 smithi195 bash[23221]: cluster 2024-03-07T03:06:37.249786+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:41.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:41 smithi148 bash[21362]: cluster 2024-03-07T03:06:39.250318+0000 mgr.y (mgr.14152) 264 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:41.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:41 smithi148 bash[31149]: cluster 2024-03-07T03:06:39.250318+0000 mgr.y (mgr.14152) 264 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:41.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:41 smithi195 bash[23221]: cluster 2024-03-07T03:06:39.250318+0000 mgr.y (mgr.14152) 264 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:43.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:43 smithi148 bash[21362]: cluster 2024-03-07T03:06:41.250881+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:43.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:43 smithi148 bash[31149]: cluster 2024-03-07T03:06:41.250881+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:43.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:43 smithi195 bash[23221]: cluster 2024-03-07T03:06:41.250881+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v243: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:45.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:45 smithi148 bash[21362]: cluster 2024-03-07T03:06:43.251453+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:45.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:45 smithi148 bash[31149]: cluster 2024-03-07T03:06:43.251453+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:45.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:45 smithi195 bash[23221]: cluster 2024-03-07T03:06:43.251453+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:47.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:47 smithi148 bash[21362]: cluster 2024-03-07T03:06:45.252013+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:47.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:47 smithi148 bash[31149]: cluster 2024-03-07T03:06:45.252013+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:47.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:47 smithi195 bash[23221]: cluster 2024-03-07T03:06:45.252013+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:48.522 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:48 smithi195 bash[23221]: audit 2024-03-07T03:06:47.412495+0000 mon.a (mon.0) 556 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:06:48.522 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:48 smithi195 bash[23221]: audit 2024-03-07T03:06:47.414387+0000 mon.a (mon.0) 557 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:48.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:48 smithi148 bash[21362]: audit 2024-03-07T03:06:47.412495+0000 mon.a (mon.0) 556 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:06:48.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:48 smithi148 bash[21362]: audit 2024-03-07T03:06:47.414387+0000 mon.a (mon.0) 557 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:48.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:48 smithi148 bash[31149]: audit 2024-03-07T03:06:47.412495+0000 mon.a (mon.0) 556 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:06:48.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:48 smithi148 bash[31149]: audit 2024-03-07T03:06:47.414387+0000 mon.a (mon.0) 557 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:49 smithi148 bash[21362]: cluster 2024-03-07T03:06:47.252574+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:49 smithi148 bash[21362]: cephadm 2024-03-07T03:06:47.415895+0000 mgr.y (mgr.14152) 269 : cephadm 1 Deploying daemon osd.6 on smithi195 2024-03-07T03:06:49.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:49 smithi148 bash[31149]: cluster 2024-03-07T03:06:47.252574+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:49.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:49 smithi148 bash[31149]: cephadm 2024-03-07T03:06:47.415895+0000 mgr.y (mgr.14152) 269 : cephadm 1 Deploying daemon osd.6 on smithi195 2024-03-07T03:06:49.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:49 smithi195 bash[23221]: cluster 2024-03-07T03:06:47.252574+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:49.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:49 smithi195 bash[23221]: cephadm 2024-03-07T03:06:47.415895+0000 mgr.y (mgr.14152) 269 : cephadm 1 Deploying daemon osd.6 on smithi195 2024-03-07T03:06:51.453 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:51.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:51 smithi195 bash[23221]: cluster 2024-03-07T03:06:49.253071+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:51.454 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:51.454 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:51.454 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:51.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:51 smithi148 bash[21362]: cluster 2024-03-07T03:06:49.253071+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:51.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:51 smithi148 bash[31149]: cluster 2024-03-07T03:06:49.253071+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:51.703 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:51.704 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:51.704 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:51.704 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:06:51 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:06:53.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[21362]: cluster 2024-03-07T03:06:51.253565+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:53.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[21362]: audit 2024-03-07T03:06:51.752707+0000 mon.a (mon.0) 558 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:53.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[21362]: audit 2024-03-07T03:06:51.756839+0000 mon.a (mon.0) 559 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:53.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[21362]: audit 2024-03-07T03:06:51.762665+0000 mon.a (mon.0) 560 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:53.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[31149]: cluster 2024-03-07T03:06:51.253565+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:53.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[31149]: audit 2024-03-07T03:06:51.752707+0000 mon.a (mon.0) 558 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:53.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[31149]: audit 2024-03-07T03:06:51.756839+0000 mon.a (mon.0) 559 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:53.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:52 smithi148 bash[31149]: audit 2024-03-07T03:06:51.762665+0000 mon.a (mon.0) 560 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:53.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:52 smithi195 bash[23221]: cluster 2024-03-07T03:06:51.253565+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:53.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:52 smithi195 bash[23221]: audit 2024-03-07T03:06:51.752707+0000 mon.a (mon.0) 558 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:53.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:52 smithi195 bash[23221]: audit 2024-03-07T03:06:51.756839+0000 mon.a (mon.0) 559 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:06:53.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:52 smithi195 bash[23221]: audit 2024-03-07T03:06:51.762665+0000 mon.a (mon.0) 560 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:55.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:55 smithi148 bash[21362]: cluster 2024-03-07T03:06:53.254118+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:55.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:55 smithi148 bash[31149]: cluster 2024-03-07T03:06:53.254118+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:55.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:55 smithi195 bash[23221]: cluster 2024-03-07T03:06:53.254118+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:57.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[21362]: cluster 2024-03-07T03:06:55.254698+0000 mgr.y (mgr.14152) 273 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:57.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[21362]: audit 2024-03-07T03:06:56.969271+0000 mon.a (mon.0) 561 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[21362]: audit 2024-03-07T03:06:56.980898+0000 mon.a (mon.0) 562 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[21362]: audit 2024-03-07T03:06:56.983037+0000 mon.a (mon.0) 563 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:57.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[21362]: audit 2024-03-07T03:06:56.984977+0000 mon.a (mon.0) 564 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:57.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[21362]: audit 2024-03-07T03:06:56.995486+0000 mon.a (mon.0) 565 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[31149]: cluster 2024-03-07T03:06:55.254698+0000 mgr.y (mgr.14152) 273 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:57.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[31149]: audit 2024-03-07T03:06:56.969271+0000 mon.a (mon.0) 561 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[31149]: audit 2024-03-07T03:06:56.980898+0000 mon.a (mon.0) 562 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[31149]: audit 2024-03-07T03:06:56.983037+0000 mon.a (mon.0) 563 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:57.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[31149]: audit 2024-03-07T03:06:56.984977+0000 mon.a (mon.0) 564 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:57.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:57 smithi148 bash[31149]: audit 2024-03-07T03:06:56.995486+0000 mon.a (mon.0) 565 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:57 smithi195 bash[23221]: cluster 2024-03-07T03:06:55.254698+0000 mgr.y (mgr.14152) 273 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:57.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:57 smithi195 bash[23221]: audit 2024-03-07T03:06:56.969271+0000 mon.a (mon.0) 561 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:57 smithi195 bash[23221]: audit 2024-03-07T03:06:56.980898+0000 mon.a (mon.0) 562 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:57.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:57 smithi195 bash[23221]: audit 2024-03-07T03:06:56.983037+0000 mon.a (mon.0) 563 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:06:57.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:57 smithi195 bash[23221]: audit 2024-03-07T03:06:56.984977+0000 mon.a (mon.0) 564 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:06:57.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:57 smithi195 bash[23221]: audit 2024-03-07T03:06:56.995486+0000 mon.a (mon.0) 565 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:06:58.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:58 smithi195 bash[23221]: audit 2024-03-07T03:06:58.083578+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:06:58.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:58 smithi195 bash[23221]: audit 2024-03-07T03:06:58.086141+0000 mon.a (mon.0) 566 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:06:58.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:58 smithi148 bash[21362]: audit 2024-03-07T03:06:58.083578+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:06:58.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:58 smithi148 bash[21362]: audit 2024-03-07T03:06:58.086141+0000 mon.a (mon.0) 566 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:06:58.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:58 smithi148 bash[31149]: audit 2024-03-07T03:06:58.083578+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:06:58.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:58 smithi148 bash[31149]: audit 2024-03-07T03:06:58.086141+0000 mon.a (mon.0) 566 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-07T03:06:59.436 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:59 smithi195 bash[23221]: cluster 2024-03-07T03:06:57.255370+0000 mgr.y (mgr.14152) 274 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:59.436 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:59 smithi195 bash[23221]: audit 2024-03-07T03:06:58.168507+0000 mon.b (mon.1) 21 : audit 1 from='osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:59.437 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:59 smithi195 bash[23221]: audit 2024-03-07T03:06:58.169046+0000 mon.a (mon.0) 567 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-07T03:06:59.437 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:59 smithi195 bash[23221]: cluster 2024-03-07T03:06:58.169216+0000 mon.a (mon.0) 568 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-03-07T03:06:59.437 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:59 smithi195 bash[23221]: audit 2024-03-07T03:06:58.169546+0000 mon.a (mon.0) 569 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:59.437 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:06:59 smithi195 bash[23221]: audit 2024-03-07T03:06:58.170839+0000 mon.a (mon.0) 570 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[21362]: cluster 2024-03-07T03:06:57.255370+0000 mgr.y (mgr.14152) 274 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[21362]: audit 2024-03-07T03:06:58.168507+0000 mon.b (mon.1) 21 : audit 1 from='osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[21362]: audit 2024-03-07T03:06:58.169046+0000 mon.a (mon.0) 567 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-07T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[21362]: cluster 2024-03-07T03:06:58.169216+0000 mon.a (mon.0) 568 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-03-07T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[21362]: audit 2024-03-07T03:06:58.169546+0000 mon.a (mon.0) 569 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:59.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[21362]: audit 2024-03-07T03:06:58.170839+0000 mon.a (mon.0) 570 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:59.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[31149]: cluster 2024-03-07T03:06:57.255370+0000 mgr.y (mgr.14152) 274 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:06:59.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[31149]: audit 2024-03-07T03:06:58.168507+0000 mon.b (mon.1) 21 : audit 1 from='osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:06:59.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[31149]: audit 2024-03-07T03:06:58.169046+0000 mon.a (mon.0) 567 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-07T03:06:59.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[31149]: cluster 2024-03-07T03:06:58.169216+0000 mon.a (mon.0) 568 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-03-07T03:06:59.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[31149]: audit 2024-03-07T03:06:58.169546+0000 mon.a (mon.0) 569 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:06:59.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:06:59 smithi148 bash[31149]: audit 2024-03-07T03:06:58.170839+0000 mon.a (mon.0) 570 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:07:00.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:00 smithi195 bash[23221]: audit 2024-03-07T03:06:59.170338+0000 mon.a (mon.0) 571 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:07:00.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:00 smithi195 bash[23221]: cluster 2024-03-07T03:06:59.170484+0000 mon.a (mon.0) 572 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-03-07T03:07:00.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:00 smithi195 bash[23221]: audit 2024-03-07T03:06:59.171200+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:00.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:00 smithi195 bash[23221]: audit 2024-03-07T03:06:59.181803+0000 mon.a (mon.0) 574 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[21362]: audit 2024-03-07T03:06:59.170338+0000 mon.a (mon.0) 571 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[21362]: cluster 2024-03-07T03:06:59.170484+0000 mon.a (mon.0) 572 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[21362]: audit 2024-03-07T03:06:59.171200+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[21362]: audit 2024-03-07T03:06:59.181803+0000 mon.a (mon.0) 574 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[31149]: audit 2024-03-07T03:06:59.170338+0000 mon.a (mon.0) 571 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[31149]: cluster 2024-03-07T03:06:59.170484+0000 mon.a (mon.0) 572 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[31149]: audit 2024-03-07T03:06:59.171200+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:00.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:00 smithi148 bash[31149]: audit 2024-03-07T03:06:59.181803+0000 mon.a (mon.0) 574 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:00.850 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 6 on host 'smithi195' 2024-03-07T03:07:01.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: cluster 2024-03-07T03:06:59.116012+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:07:01.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: cluster 2024-03-07T03:06:59.116120+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:07:01.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: cluster 2024-03-07T03:06:59.255949+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:07:01.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: audit 2024-03-07T03:07:00.180380+0000 mon.a (mon.0) 575 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:01.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: cluster 2024-03-07T03:07:00.191170+0000 mon.a (mon.0) 576 : cluster 1 osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669] boot 2024-03-07T03:07:01.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: cluster 2024-03-07T03:07:00.191301+0000 mon.a (mon.0) 577 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-03-07T03:07:01.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: audit 2024-03-07T03:07:00.192086+0000 mon.a (mon.0) 578 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: audit 2024-03-07T03:07:00.807518+0000 mon.a (mon.0) 579 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: audit 2024-03-07T03:07:00.819254+0000 mon.a (mon.0) 580 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[21362]: audit 2024-03-07T03:07:00.842779+0000 mon.a (mon.0) 581 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: cluster 2024-03-07T03:06:59.116012+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: cluster 2024-03-07T03:06:59.116120+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: cluster 2024-03-07T03:06:59.255949+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: audit 2024-03-07T03:07:00.180380+0000 mon.a (mon.0) 575 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: cluster 2024-03-07T03:07:00.191170+0000 mon.a (mon.0) 576 : cluster 1 osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669] boot 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: cluster 2024-03-07T03:07:00.191301+0000 mon.a (mon.0) 577 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: audit 2024-03-07T03:07:00.192086+0000 mon.a (mon.0) 578 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: audit 2024-03-07T03:07:00.807518+0000 mon.a (mon.0) 579 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: audit 2024-03-07T03:07:00.819254+0000 mon.a (mon.0) 580 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:01.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:01 smithi148 bash[31149]: audit 2024-03-07T03:07:00.842779+0000 mon.a (mon.0) 581 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:01.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: cluster 2024-03-07T03:06:59.116012+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: cluster 2024-03-07T03:06:59.116120+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: cluster 2024-03-07T03:06:59.255949+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: audit 2024-03-07T03:07:00.180380+0000 mon.a (mon.0) 575 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: cluster 2024-03-07T03:07:00.191170+0000 mon.a (mon.0) 576 : cluster 1 osd.6 [v2:172.21.15.195:6816/4169107669,v1:172.21.15.195:6817/4169107669] boot 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: cluster 2024-03-07T03:07:00.191301+0000 mon.a (mon.0) 577 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: audit 2024-03-07T03:07:00.192086+0000 mon.a (mon.0) 578 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: audit 2024-03-07T03:07:00.807518+0000 mon.a (mon.0) 579 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: audit 2024-03-07T03:07:00.819254+0000 mon.a (mon.0) 580 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:01.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:01 smithi195 bash[23221]: audit 2024-03-07T03:07:00.842779+0000 mon.a (mon.0) 581 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:01.580 DEBUG:teuthology.orchestra.run.smithi195:osd.6> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.6.service 2024-03-07T03:07:01.583 INFO:tasks.cephadm:Deploying osd.7 on smithi195 with /dev/vg_nvme/lv_1... 2024-03-07T03:07:01.583 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-07T03:07:02.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:02 smithi148 bash[21362]: cluster 2024-03-07T03:07:01.204199+0000 mon.a (mon.0) 582 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-03-07T03:07:02.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:02 smithi148 bash[31149]: cluster 2024-03-07T03:07:01.204199+0000 mon.a (mon.0) 582 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-03-07T03:07:02.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:02 smithi195 bash[23221]: cluster 2024-03-07T03:07:01.204199+0000 mon.a (mon.0) 582 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-03-07T03:07:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:03 smithi148 bash[21362]: cluster 2024-03-07T03:07:01.256520+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:03 smithi148 bash[21362]: cluster 2024-03-07T03:07:02.216558+0000 mon.a (mon.0) 583 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-03-07T03:07:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:03 smithi148 bash[21362]: cluster 2024-03-07T03:07:03.154270+0000 mon.a (mon.0) 584 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-03-07T03:07:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:03 smithi148 bash[31149]: cluster 2024-03-07T03:07:01.256520+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:03 smithi148 bash[31149]: cluster 2024-03-07T03:07:02.216558+0000 mon.a (mon.0) 583 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-03-07T03:07:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:03 smithi148 bash[31149]: cluster 2024-03-07T03:07:03.154270+0000 mon.a (mon.0) 584 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-03-07T03:07:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:03 smithi195 bash[23221]: cluster 2024-03-07T03:07:01.256520+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v257: 1 pgs: 1 active+clean; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:03 smithi195 bash[23221]: cluster 2024-03-07T03:07:02.216558+0000 mon.a (mon.0) 583 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-03-07T03:07:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:03 smithi195 bash[23221]: cluster 2024-03-07T03:07:03.154270+0000 mon.a (mon.0) 584 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-03-07T03:07:05.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:05 smithi148 bash[21362]: cluster 2024-03-07T03:07:03.257033+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v260: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:05.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:05 smithi148 bash[21362]: cluster 2024-03-07T03:07:04.167934+0000 mon.a (mon.0) 585 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-03-07T03:07:05.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:05 smithi148 bash[31149]: cluster 2024-03-07T03:07:03.257033+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v260: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:05.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:05 smithi148 bash[31149]: cluster 2024-03-07T03:07:04.167934+0000 mon.a (mon.0) 585 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-03-07T03:07:05.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:05 smithi195 bash[23221]: cluster 2024-03-07T03:07:03.257033+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v260: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:05.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:05 smithi195 bash[23221]: cluster 2024-03-07T03:07:04.167934+0000 mon.a (mon.0) 585 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-03-07T03:07:05.680 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:07:07.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:07 smithi148 bash[21362]: cluster 2024-03-07T03:07:05.257592+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v262: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:07.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:07 smithi148 bash[31149]: cluster 2024-03-07T03:07:05.257592+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v262: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:07.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:07 smithi195 bash[23221]: cluster 2024-03-07T03:07:05.257592+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v262: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:09.445 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:09 smithi195 bash[23221]: cluster 2024-03-07T03:07:07.258196+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:07:09.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:09 smithi148 bash[21362]: cluster 2024-03-07T03:07:07.258196+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:07:09.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:09 smithi148 bash[31149]: cluster 2024-03-07T03:07:07.258196+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v263: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-07T03:07:10.114 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-07T03:07:10.114 INFO:teuthology.orchestra.run.smithi195.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-07T03:07:10.114 INFO:teuthology.orchestra.run.smithi195.stderr: stderr: 10+0 records in 2024-03-07T03:07:10.114 INFO:teuthology.orchestra.run.smithi195.stderr:10+0 records out 2024-03-07T03:07:10.115 INFO:teuthology.orchestra.run.smithi195.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0101925 s, 1.0 GB/s 2024-03-07T03:07:10.115 INFO:teuthology.orchestra.run.smithi195.stderr:--> Zapping successful for: 2024-03-07T03:07:11.114 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch daemon add osd smithi195:vg_nvme/lv_1 2024-03-07T03:07:11.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:11 smithi195 bash[23221]: cluster 2024-03-07T03:07:09.258812+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-03-07T03:07:11.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:11 smithi148 bash[21362]: cluster 2024-03-07T03:07:09.258812+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-03-07T03:07:11.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:11 smithi148 bash[31149]: cluster 2024-03-07T03:07:09.258812+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v264: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: cluster 2024-03-07T03:07:11.259415+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: cephadm 2024-03-07T03:07:11.288969+0000 mgr.y (mgr.14152) 282 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.299498+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.311506+0000 mon.a (mon.0) 587 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.314008+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.315790+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.317467+0000 mon.a (mon.0) 590 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: cephadm 2024-03-07T03:07:11.318748+0000 mgr.y (mgr.14152) 283 : cephadm 1 Adjusting osd_memory_target on smithi195 to 5755M 2024-03-07T03:07:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.329233+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.331399+0000 mon.a (mon.0) 592 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.333003+0000 mon.a (mon.0) 593 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[21362]: audit 2024-03-07T03:07:11.344750+0000 mon.a (mon.0) 594 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: cluster 2024-03-07T03:07:11.259415+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: cephadm 2024-03-07T03:07:11.288969+0000 mgr.y (mgr.14152) 282 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.299498+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.311506+0000 mon.a (mon.0) 587 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.314008+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.315790+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.317467+0000 mon.a (mon.0) 590 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: cephadm 2024-03-07T03:07:11.318748+0000 mgr.y (mgr.14152) 283 : cephadm 1 Adjusting osd_memory_target on smithi195 to 5755M 2024-03-07T03:07:12.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.329233+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.331399+0000 mon.a (mon.0) 592 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:12.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.333003+0000 mon.a (mon.0) 593 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:12.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:12 smithi148 bash[31149]: audit 2024-03-07T03:07:11.344750+0000 mon.a (mon.0) 594 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: cluster 2024-03-07T03:07:11.259415+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v265: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-03-07T03:07:12.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: cephadm 2024-03-07T03:07:11.288969+0000 mgr.y (mgr.14152) 282 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:07:12.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.299498+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.311506+0000 mon.a (mon.0) 587 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.314008+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.315790+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.317467+0000 mon.a (mon.0) 590 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: cephadm 2024-03-07T03:07:11.318748+0000 mgr.y (mgr.14152) 283 : cephadm 1 Adjusting osd_memory_target on smithi195 to 5755M 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.329233+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.331399+0000 mon.a (mon.0) 592 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.333003+0000 mon.a (mon.0) 593 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:12 smithi195 bash[23221]: audit 2024-03-07T03:07:11.344750+0000 mon.a (mon.0) 594 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:15.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:15 smithi148 bash[21362]: cluster 2024-03-07T03:07:13.260035+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:07:15.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:15 smithi148 bash[31149]: cluster 2024-03-07T03:07:13.260035+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:07:15.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:15 smithi195 bash[23221]: cluster 2024-03-07T03:07:13.260035+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v266: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-07T03:07:15.810 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:07:17.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:17 smithi195 bash[23221]: cluster 2024-03-07T03:07:15.260521+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:07:17.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:17 smithi148 bash[21362]: cluster 2024-03-07T03:07:15.260521+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:07:17.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:17 smithi148 bash[31149]: cluster 2024-03-07T03:07:15.260521+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[21362]: cluster 2024-03-07T03:07:17.261096+0000 mgr.y (mgr.14152) 286 : cluster 0 pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[21362]: audit 2024-03-07T03:07:18.702421+0000 mon.a (mon.0) 595 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[21362]: audit 2024-03-07T03:07:18.708785+0000 mon.a (mon.0) 596 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[21362]: audit 2024-03-07T03:07:18.710635+0000 mon.a (mon.0) 597 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[31149]: cluster 2024-03-07T03:07:17.261096+0000 mgr.y (mgr.14152) 286 : cluster 0 pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[31149]: audit 2024-03-07T03:07:18.702421+0000 mon.a (mon.0) 595 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[31149]: audit 2024-03-07T03:07:18.708785+0000 mon.a (mon.0) 596 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:07:19.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:19 smithi148 bash[31149]: audit 2024-03-07T03:07:18.710635+0000 mon.a (mon.0) 597 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:19.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:19 smithi195 bash[23221]: cluster 2024-03-07T03:07:17.261096+0000 mgr.y (mgr.14152) 286 : cluster 0 pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:07:19.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:19 smithi195 bash[23221]: audit 2024-03-07T03:07:18.702421+0000 mon.a (mon.0) 595 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-07T03:07:19.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:19 smithi195 bash[23221]: audit 2024-03-07T03:07:18.708785+0000 mon.a (mon.0) 596 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-07T03:07:19.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:19 smithi195 bash[23221]: audit 2024-03-07T03:07:18.710635+0000 mon.a (mon.0) 597 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:20.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:20 smithi148 bash[21362]: audit 2024-03-07T03:07:18.695640+0000 mgr.y (mgr.14152) 287 : audit 0 from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:07:20.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:20 smithi148 bash[31149]: audit 2024-03-07T03:07:18.695640+0000 mgr.y (mgr.14152) 287 : audit 0 from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:07:20.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:20 smithi195 bash[23221]: audit 2024-03-07T03:07:18.695640+0000 mgr.y (mgr.14152) 287 : audit 0 from='client.24256 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi195:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:07:21.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:21 smithi148 bash[21362]: cluster 2024-03-07T03:07:19.261730+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:21.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:21 smithi148 bash[31149]: cluster 2024-03-07T03:07:19.261730+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:21.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:21 smithi195 bash[23221]: cluster 2024-03-07T03:07:19.261730+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:23.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:23 smithi148 bash[21362]: cluster 2024-03-07T03:07:21.262421+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:23.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:23 smithi148 bash[31149]: cluster 2024-03-07T03:07:21.262421+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:23.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:23 smithi195 bash[23221]: cluster 2024-03-07T03:07:21.262421+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:25.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:25 smithi148 bash[21362]: cluster 2024-03-07T03:07:23.263105+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:25.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:25 smithi148 bash[31149]: cluster 2024-03-07T03:07:23.263105+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:25.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:25 smithi195 bash[23221]: cluster 2024-03-07T03:07:23.263105+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[21362]: cluster 2024-03-07T03:07:25.263789+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[21362]: audit 2024-03-07T03:07:27.084715+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.195:0/3175772544' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]: dispatch 2024-03-07T03:07:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[21362]: audit 2024-03-07T03:07:27.087247+0000 mon.a (mon.0) 598 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]: dispatch 2024-03-07T03:07:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[21362]: audit 2024-03-07T03:07:27.100821+0000 mon.a (mon.0) 599 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]': finished 2024-03-07T03:07:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[21362]: cluster 2024-03-07T03:07:27.100987+0000 mon.a (mon.0) 600 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-03-07T03:07:27.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[21362]: audit 2024-03-07T03:07:27.101228+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:27.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[31149]: cluster 2024-03-07T03:07:25.263789+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[31149]: audit 2024-03-07T03:07:27.084715+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.195:0/3175772544' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]: dispatch 2024-03-07T03:07:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[31149]: audit 2024-03-07T03:07:27.087247+0000 mon.a (mon.0) 598 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]: dispatch 2024-03-07T03:07:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[31149]: audit 2024-03-07T03:07:27.100821+0000 mon.a (mon.0) 599 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]': finished 2024-03-07T03:07:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[31149]: cluster 2024-03-07T03:07:27.100987+0000 mon.a (mon.0) 600 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-03-07T03:07:27.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:27 smithi148 bash[31149]: audit 2024-03-07T03:07:27.101228+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:27.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:27 smithi195 bash[23221]: cluster 2024-03-07T03:07:25.263789+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:27.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:27 smithi195 bash[23221]: audit 2024-03-07T03:07:27.084715+0000 mon.b (mon.1) 22 : audit 1 from='client.? 172.21.15.195:0/3175772544' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]: dispatch 2024-03-07T03:07:27.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:27 smithi195 bash[23221]: audit 2024-03-07T03:07:27.087247+0000 mon.a (mon.0) 598 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]: dispatch 2024-03-07T03:07:27.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:27 smithi195 bash[23221]: audit 2024-03-07T03:07:27.100821+0000 mon.a (mon.0) 599 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6c01c7ac-d80e-48bb-a060-7dc2e2d22e41"}]': finished 2024-03-07T03:07:27.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:27 smithi195 bash[23221]: cluster 2024-03-07T03:07:27.100987+0000 mon.a (mon.0) 600 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-03-07T03:07:27.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:27 smithi195 bash[23221]: audit 2024-03-07T03:07:27.101228+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:28.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:28 smithi148 bash[21362]: audit 2024-03-07T03:07:27.843982+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.195:0/3249538868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:07:28.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:28 smithi148 bash[31149]: audit 2024-03-07T03:07:27.843982+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.195:0/3249538868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:07:28.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:28 smithi195 bash[23221]: audit 2024-03-07T03:07:27.843982+0000 mon.b (mon.1) 23 : audit 0 from='client.? 172.21.15.195:0/3249538868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-07T03:07:29.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:29 smithi148 bash[21362]: cluster 2024-03-07T03:07:27.264394+0000 mgr.y (mgr.14152) 292 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:29.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:29 smithi148 bash[31149]: cluster 2024-03-07T03:07:27.264394+0000 mgr.y (mgr.14152) 292 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:29.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:29 smithi195 bash[23221]: cluster 2024-03-07T03:07:27.264394+0000 mgr.y (mgr.14152) 292 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:31.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:31 smithi148 bash[21362]: cluster 2024-03-07T03:07:29.264959+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:31.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:31 smithi148 bash[31149]: cluster 2024-03-07T03:07:29.264959+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:31.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:31 smithi195 bash[23221]: cluster 2024-03-07T03:07:29.264959+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:33.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:33 smithi148 bash[21362]: cluster 2024-03-07T03:07:31.265540+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:33.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:33 smithi148 bash[31149]: cluster 2024-03-07T03:07:31.265540+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:33.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:33 smithi195 bash[23221]: cluster 2024-03-07T03:07:31.265540+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:35.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:35 smithi148 bash[21362]: cluster 2024-03-07T03:07:33.266136+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:35.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:35 smithi148 bash[31149]: cluster 2024-03-07T03:07:33.266136+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:35.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:35 smithi195 bash[23221]: cluster 2024-03-07T03:07:33.266136+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:37.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:37 smithi195 bash[23221]: cluster 2024-03-07T03:07:35.266694+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:37.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:37 smithi148 bash[21362]: cluster 2024-03-07T03:07:35.266694+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:37.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:37 smithi148 bash[31149]: cluster 2024-03-07T03:07:35.266694+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v278: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:39.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:39 smithi148 bash[21362]: cluster 2024-03-07T03:07:37.267294+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:39.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:39 smithi148 bash[31149]: cluster 2024-03-07T03:07:37.267294+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:39.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:39 smithi195 bash[23221]: cluster 2024-03-07T03:07:37.267294+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:40.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:40 smithi148 bash[21362]: audit 2024-03-07T03:07:39.631351+0000 mon.a (mon.0) 602 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:07:40.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:40 smithi148 bash[21362]: audit 2024-03-07T03:07:39.633125+0000 mon.a (mon.0) 603 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:40 smithi148 bash[31149]: audit 2024-03-07T03:07:39.631351+0000 mon.a (mon.0) 602 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:07:40.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:40 smithi148 bash[31149]: audit 2024-03-07T03:07:39.633125+0000 mon.a (mon.0) 603 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:40.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:40 smithi195 bash[23221]: audit 2024-03-07T03:07:39.631351+0000 mon.a (mon.0) 602 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:07:40.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:40 smithi195 bash[23221]: audit 2024-03-07T03:07:39.633125+0000 mon.a (mon.0) 603 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:41.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:41 smithi148 bash[21362]: cluster 2024-03-07T03:07:39.267813+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:41.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:41 smithi148 bash[21362]: cephadm 2024-03-07T03:07:39.634710+0000 mgr.y (mgr.14152) 299 : cephadm 1 Deploying daemon osd.7 on smithi195 2024-03-07T03:07:41.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:41 smithi148 bash[31149]: cluster 2024-03-07T03:07:39.267813+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:41.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:41 smithi148 bash[31149]: cephadm 2024-03-07T03:07:39.634710+0000 mgr.y (mgr.14152) 299 : cephadm 1 Deploying daemon osd.7 on smithi195 2024-03-07T03:07:41.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:41 smithi195 bash[23221]: cluster 2024-03-07T03:07:39.267813+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:41.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:41 smithi195 bash[23221]: cephadm 2024-03-07T03:07:39.634710+0000 mgr.y (mgr.14152) 299 : cephadm 1 Deploying daemon osd.7 on smithi195 2024-03-07T03:07:43.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:43 smithi148 bash[21362]: cluster 2024-03-07T03:07:41.268443+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:43.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:43 smithi148 bash[31149]: cluster 2024-03-07T03:07:41.268443+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:43.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:43 smithi195 bash[23221]: cluster 2024-03-07T03:07:41.268443+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:43.878 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:43.879 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:07:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:43.879 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:07:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:43.879 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:07:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:43.880 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:07:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:44.161 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:44.161 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:07:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:44.161 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:07:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:44.162 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:07:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:44.162 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:07:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:07:45.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[31149]: cluster 2024-03-07T03:07:43.268965+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:45.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[31149]: audit 2024-03-07T03:07:44.164645+0000 mon.a (mon.0) 604 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:45.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[31149]: audit 2024-03-07T03:07:44.174465+0000 mon.a (mon.0) 605 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:45.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[31149]: audit 2024-03-07T03:07:44.175281+0000 mon.a (mon.0) 606 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:45.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[21362]: cluster 2024-03-07T03:07:43.268965+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:45.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[21362]: audit 2024-03-07T03:07:44.164645+0000 mon.a (mon.0) 604 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:45.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[21362]: audit 2024-03-07T03:07:44.174465+0000 mon.a (mon.0) 605 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:45.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:45 smithi148 bash[21362]: audit 2024-03-07T03:07:44.175281+0000 mon.a (mon.0) 606 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:45.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:45 smithi195 bash[23221]: cluster 2024-03-07T03:07:43.268965+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:45.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:45 smithi195 bash[23221]: audit 2024-03-07T03:07:44.164645+0000 mon.a (mon.0) 604 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:45.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:45 smithi195 bash[23221]: audit 2024-03-07T03:07:44.174465+0000 mon.a (mon.0) 605 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:45.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:45 smithi195 bash[23221]: audit 2024-03-07T03:07:44.175281+0000 mon.a (mon.0) 606 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:47.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:47 smithi148 bash[21362]: cluster 2024-03-07T03:07:45.269557+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:47.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:47 smithi148 bash[31149]: cluster 2024-03-07T03:07:45.269557+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:47.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:47 smithi195 bash[23221]: cluster 2024-03-07T03:07:45.269557+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:49 smithi148 bash[21362]: cluster 2024-03-07T03:07:47.270076+0000 mgr.y (mgr.14152) 303 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:49.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:49 smithi148 bash[31149]: cluster 2024-03-07T03:07:47.270076+0000 mgr.y (mgr.14152) 303 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:49.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:49 smithi195 bash[23221]: cluster 2024-03-07T03:07:47.270076+0000 mgr.y (mgr.14152) 303 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:50.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:50 smithi195 bash[23221]: cluster 2024-03-07T03:07:49.270635+0000 mgr.y (mgr.14152) 304 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:50.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:50 smithi195 bash[23221]: audit 2024-03-07T03:07:49.426264+0000 mon.a (mon.0) 607 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:50 smithi195 bash[23221]: audit 2024-03-07T03:07:49.436797+0000 mon.a (mon.0) 608 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:50 smithi195 bash[23221]: audit 2024-03-07T03:07:49.982790+0000 mon.a (mon.0) 609 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:50.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:50 smithi195 bash[23221]: audit 2024-03-07T03:07:49.984773+0000 mon.a (mon.0) 610 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:50.783 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:50 smithi195 bash[23221]: audit 2024-03-07T03:07:49.995149+0000 mon.a (mon.0) 611 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[21362]: cluster 2024-03-07T03:07:49.270635+0000 mgr.y (mgr.14152) 304 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:50.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[21362]: audit 2024-03-07T03:07:49.426264+0000 mon.a (mon.0) 607 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[21362]: audit 2024-03-07T03:07:49.436797+0000 mon.a (mon.0) 608 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[21362]: audit 2024-03-07T03:07:49.982790+0000 mon.a (mon.0) 609 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:50.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[21362]: audit 2024-03-07T03:07:49.984773+0000 mon.a (mon.0) 610 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:50.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[21362]: audit 2024-03-07T03:07:49.995149+0000 mon.a (mon.0) 611 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[31149]: cluster 2024-03-07T03:07:49.270635+0000 mgr.y (mgr.14152) 304 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:50.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[31149]: audit 2024-03-07T03:07:49.426264+0000 mon.a (mon.0) 607 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[31149]: audit 2024-03-07T03:07:49.436797+0000 mon.a (mon.0) 608 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:50.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[31149]: audit 2024-03-07T03:07:49.982790+0000 mon.a (mon.0) 609 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:07:50.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[31149]: audit 2024-03-07T03:07:49.984773+0000 mon.a (mon.0) 610 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:07:50.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:50 smithi148 bash[31149]: audit 2024-03-07T03:07:49.995149+0000 mon.a (mon.0) 611 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:51.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:51 smithi148 bash[21362]: audit 2024-03-07T03:07:50.792445+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:07:51.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:51 smithi148 bash[21362]: audit 2024-03-07T03:07:50.794680+0000 mon.a (mon.0) 612 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:07:51.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:51 smithi148 bash[31149]: audit 2024-03-07T03:07:50.792445+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:07:51.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:51 smithi148 bash[31149]: audit 2024-03-07T03:07:50.794680+0000 mon.a (mon.0) 612 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:07:51.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:51 smithi195 bash[23221]: audit 2024-03-07T03:07:50.792445+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:07:51.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:51 smithi195 bash[23221]: audit 2024-03-07T03:07:50.794680+0000 mon.a (mon.0) 612 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-07T03:07:52.715 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:52 smithi195 bash[23221]: cluster 2024-03-07T03:07:51.271209+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:52.715 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:52 smithi195 bash[23221]: audit 2024-03-07T03:07:51.439021+0000 mon.b (mon.1) 25 : audit 1 from='osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:07:52.715 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:52 smithi195 bash[23221]: audit 2024-03-07T03:07:51.439043+0000 mon.a (mon.0) 613 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-07T03:07:52.715 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:52 smithi195 bash[23221]: cluster 2024-03-07T03:07:51.439221+0000 mon.a (mon.0) 614 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-03-07T03:07:52.715 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:52 smithi195 bash[23221]: audit 2024-03-07T03:07:51.439581+0000 mon.a (mon.0) 615 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:52.716 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:52 smithi195 bash[23221]: audit 2024-03-07T03:07:51.441368+0000 mon.a (mon.0) 616 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:07:52.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[21362]: cluster 2024-03-07T03:07:51.271209+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:52.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[21362]: audit 2024-03-07T03:07:51.439021+0000 mon.b (mon.1) 25 : audit 1 from='osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:07:52.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[21362]: audit 2024-03-07T03:07:51.439043+0000 mon.a (mon.0) 613 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-07T03:07:52.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[21362]: cluster 2024-03-07T03:07:51.439221+0000 mon.a (mon.0) 614 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-03-07T03:07:52.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[21362]: audit 2024-03-07T03:07:51.439581+0000 mon.a (mon.0) 615 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:52.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[21362]: audit 2024-03-07T03:07:51.441368+0000 mon.a (mon.0) 616 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:07:52.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[31149]: cluster 2024-03-07T03:07:51.271209+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:52.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[31149]: audit 2024-03-07T03:07:51.439021+0000 mon.b (mon.1) 25 : audit 1 from='osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:07:52.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[31149]: audit 2024-03-07T03:07:51.439043+0000 mon.a (mon.0) 613 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-07T03:07:52.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[31149]: cluster 2024-03-07T03:07:51.439221+0000 mon.a (mon.0) 614 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-03-07T03:07:52.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[31149]: audit 2024-03-07T03:07:51.439581+0000 mon.a (mon.0) 615 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:52.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:52 smithi148 bash[31149]: audit 2024-03-07T03:07:51.441368+0000 mon.a (mon.0) 616 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:07:53.226 INFO:teuthology.orchestra.run.smithi195.stdout:Created osd(s) 7 on host 'smithi195' 2024-03-07T03:07:53.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: audit 2024-03-07T03:07:52.441571+0000 mon.a (mon.0) 617 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:07:53.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: cluster 2024-03-07T03:07:52.441795+0000 mon.a (mon.0) 618 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: audit 2024-03-07T03:07:52.442416+0000 mon.a (mon.0) 619 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: audit 2024-03-07T03:07:52.452222+0000 mon.a (mon.0) 620 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: cluster 2024-03-07T03:07:53.172613+0000 mon.a (mon.0) 621 : cluster 1 osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566] boot 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: cluster 2024-03-07T03:07:53.172734+0000 mon.a (mon.0) 622 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: audit 2024-03-07T03:07:53.173708+0000 mon.a (mon.0) 623 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: audit 2024-03-07T03:07:53.218697+0000 mon.a (mon.0) 624 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: audit 2024-03-07T03:07:53.229197+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:53.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:53 smithi195 bash[23221]: audit 2024-03-07T03:07:53.238808+0000 mon.a (mon.0) 626 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:53.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: audit 2024-03-07T03:07:52.441571+0000 mon.a (mon.0) 617 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:07:53.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: cluster 2024-03-07T03:07:52.441795+0000 mon.a (mon.0) 618 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-03-07T03:07:53.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: audit 2024-03-07T03:07:52.442416+0000 mon.a (mon.0) 619 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: audit 2024-03-07T03:07:52.452222+0000 mon.a (mon.0) 620 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: cluster 2024-03-07T03:07:53.172613+0000 mon.a (mon.0) 621 : cluster 1 osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566] boot 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: cluster 2024-03-07T03:07:53.172734+0000 mon.a (mon.0) 622 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: audit 2024-03-07T03:07:53.173708+0000 mon.a (mon.0) 623 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: audit 2024-03-07T03:07:53.218697+0000 mon.a (mon.0) 624 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: audit 2024-03-07T03:07:53.229197+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[21362]: audit 2024-03-07T03:07:53.238808+0000 mon.a (mon.0) 626 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: audit 2024-03-07T03:07:52.441571+0000 mon.a (mon.0) 617 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]': finished 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: cluster 2024-03-07T03:07:52.441795+0000 mon.a (mon.0) 618 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: audit 2024-03-07T03:07:52.442416+0000 mon.a (mon.0) 619 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: audit 2024-03-07T03:07:52.452222+0000 mon.a (mon.0) 620 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: cluster 2024-03-07T03:07:53.172613+0000 mon.a (mon.0) 621 : cluster 1 osd.7 [v2:172.21.15.195:6824/2298968566,v1:172.21.15.195:6825/2298968566] boot 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: cluster 2024-03-07T03:07:53.172734+0000 mon.a (mon.0) 622 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-03-07T03:07:53.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: audit 2024-03-07T03:07:53.173708+0000 mon.a (mon.0) 623 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:07:53.798 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: audit 2024-03-07T03:07:53.218697+0000 mon.a (mon.0) 624 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:53.798 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: audit 2024-03-07T03:07:53.229197+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:07:53.798 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:53 smithi148 bash[31149]: audit 2024-03-07T03:07:53.238808+0000 mon.a (mon.0) 626 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:07:54.005 DEBUG:teuthology.orchestra.run.smithi195:osd.7> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.7.service 2024-03-07T03:07:54.007 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-07T03:07:54.008 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd stat -f json 2024-03-07T03:07:54.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[21362]: cluster 2024-03-07T03:07:51.810678+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:07:54.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[21362]: cluster 2024-03-07T03:07:51.810767+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:07:54.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[21362]: cluster 2024-03-07T03:07:53.271593+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:54.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[21362]: cluster 2024-03-07T03:07:54.181855+0000 mon.a (mon.0) 627 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-03-07T03:07:54.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[31149]: cluster 2024-03-07T03:07:51.810678+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:07:54.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[31149]: cluster 2024-03-07T03:07:51.810767+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:07:54.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[31149]: cluster 2024-03-07T03:07:53.271593+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:54.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:54 smithi148 bash[31149]: cluster 2024-03-07T03:07:54.181855+0000 mon.a (mon.0) 627 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-03-07T03:07:54.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:54 smithi195 bash[23221]: cluster 2024-03-07T03:07:51.810678+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-03-07T03:07:54.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:54 smithi195 bash[23221]: cluster 2024-03-07T03:07:51.810767+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-03-07T03:07:54.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:54 smithi195 bash[23221]: cluster 2024-03-07T03:07:53.271593+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-03-07T03:07:54.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:54 smithi195 bash[23221]: cluster 2024-03-07T03:07:54.181855+0000 mon.a (mon.0) 627 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-03-07T03:07:56.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:56 smithi148 bash[21362]: cluster 2024-03-07T03:07:55.186675+0000 mon.a (mon.0) 628 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-03-07T03:07:56.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:56 smithi148 bash[31149]: cluster 2024-03-07T03:07:55.186675+0000 mon.a (mon.0) 628 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-03-07T03:07:56.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:56 smithi195 bash[23221]: cluster 2024-03-07T03:07:55.186675+0000 mon.a (mon.0) 628 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-03-07T03:07:57.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:57 smithi148 bash[21362]: cluster 2024-03-07T03:07:55.272114+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:57.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:57 smithi148 bash[21362]: cluster 2024-03-07T03:07:56.190232+0000 mon.a (mon.0) 629 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-03-07T03:07:57.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:57 smithi148 bash[31149]: cluster 2024-03-07T03:07:55.272114+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:57.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:57 smithi148 bash[31149]: cluster 2024-03-07T03:07:56.190232+0000 mon.a (mon.0) 629 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-03-07T03:07:57.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:57 smithi195 bash[23221]: cluster 2024-03-07T03:07:55.272114+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v293: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:57.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:57 smithi195 bash[23221]: cluster 2024-03-07T03:07:56.190232+0000 mon.a (mon.0) 629 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-03-07T03:07:58.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:58 smithi148 bash[21362]: cluster 2024-03-07T03:07:57.202173+0000 mon.a (mon.0) 630 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-03-07T03:07:58.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:58 smithi148 bash[31149]: cluster 2024-03-07T03:07:57.202173+0000 mon.a (mon.0) 630 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-03-07T03:07:58.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:58 smithi195 bash[23221]: cluster 2024-03-07T03:07:57.202173+0000 mon.a (mon.0) 630 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-03-07T03:07:58.728 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:07:59.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:07:59 smithi195 bash[23221]: cluster 2024-03-07T03:07:57.272680+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:59.515 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:07:59 smithi148 bash[21362]: cluster 2024-03-07T03:07:57.272680+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:07:59.515 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:07:59 smithi148 bash[31149]: cluster 2024-03-07T03:07:57.272680+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v296: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:01.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:01 smithi148 bash[21362]: cluster 2024-03-07T03:07:59.273297+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-03-07T03:08:01.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:01 smithi148 bash[31149]: cluster 2024-03-07T03:07:59.273297+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-03-07T03:08:01.394 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:08:01.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:01 smithi195 bash[23221]: cluster 2024-03-07T03:07:59.273297+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 113 KiB/s, 0 objects/s recovering 2024-03-07T03:08:02.035 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1709780873,"num_in_osds":8,"osd_in_since":1709780847,"num_remapped_pgs":0} 2024-03-07T03:08:02.036 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:08:02.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:02 smithi148 bash[31149]: audit 2024-03-07T03:08:01.402186+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.148:0/1594647339' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-07T03:08:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:02 smithi148 bash[21362]: audit 2024-03-07T03:08:01.402186+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.148:0/1594647339' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-07T03:08:02.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:02 smithi195 bash[23221]: audit 2024-03-07T03:08:01.402186+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.148:0/1594647339' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-07T03:08:03.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: cluster 2024-03-07T03:08:01.273874+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-03-07T03:08:03.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.613950+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.624142+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.626789+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.628597+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.630376+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.632169+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.642178+0000 mon.a (mon.0) 637 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.644079+0000 mon.a (mon.0) 638 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.645894+0000 mon.a (mon.0) 639 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[21362]: audit 2024-03-07T03:08:02.655671+0000 mon.a (mon.0) 640 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: cluster 2024-03-07T03:08:01.273874+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.613950+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.624142+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.626789+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.628597+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.630376+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.632169+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.642178+0000 mon.a (mon.0) 637 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.644079+0000 mon.a (mon.0) 638 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:03.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.645894+0000 mon.a (mon.0) 639 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:03.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:03 smithi148 bash[31149]: audit 2024-03-07T03:08:02.655671+0000 mon.a (mon.0) 640 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: cluster 2024-03-07T03:08:01.273874+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v298: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s, 0 objects/s recovering 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.613950+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.624142+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.626789+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.628597+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.630376+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.632169+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.642178+0000 mon.a (mon.0) 637 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.644079+0000 mon.a (mon.0) 638 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.645894+0000 mon.a (mon.0) 639 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:03.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:03 smithi195 bash[23221]: audit 2024-03-07T03:08:02.655671+0000 mon.a (mon.0) 640 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:04.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:04 smithi148 bash[31149]: cephadm 2024-03-07T03:08:02.604214+0000 mgr.y (mgr.14152) 311 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:08:04.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:04 smithi148 bash[31149]: cephadm 2024-03-07T03:08:02.633667+0000 mgr.y (mgr.14152) 312 : cephadm 1 Adjusting osd_memory_target on smithi195 to 4316M 2024-03-07T03:08:04.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:04 smithi148 bash[21362]: cephadm 2024-03-07T03:08:02.604214+0000 mgr.y (mgr.14152) 311 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:08:04.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:04 smithi148 bash[21362]: cephadm 2024-03-07T03:08:02.633667+0000 mgr.y (mgr.14152) 312 : cephadm 1 Adjusting osd_memory_target on smithi195 to 4316M 2024-03-07T03:08:04.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:04 smithi195 bash[23221]: cephadm 2024-03-07T03:08:02.604214+0000 mgr.y (mgr.14152) 311 : cephadm 1 Detected new or changed devices on smithi195 2024-03-07T03:08:04.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:04 smithi195 bash[23221]: cephadm 2024-03-07T03:08:02.633667+0000 mgr.y (mgr.14152) 312 : cephadm 1 Adjusting osd_memory_target on smithi195 to 4316M 2024-03-07T03:08:05.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:05 smithi148 bash[21362]: cluster 2024-03-07T03:08:03.274492+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:08:05.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:05 smithi148 bash[31149]: cluster 2024-03-07T03:08:03.274492+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:08:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:05 smithi195 bash[23221]: cluster 2024-03-07T03:08:03.274492+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v299: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-07T03:08:06.747 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:08:07.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:07 smithi148 bash[21362]: cluster 2024-03-07T03:08:05.275089+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-07T03:08:07.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:07 smithi148 bash[31149]: cluster 2024-03-07T03:08:05.275089+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-07T03:08:07.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:07 smithi195 bash[23221]: cluster 2024-03-07T03:08:05.275089+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v300: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-07T03:08:09.484 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:08:09.484 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":59,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:07:57.192940+0000","last_up_change":"2024-03-07T03:07:53.161981+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6803","nonce":1285694746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6805","nonce":1285694746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6809","nonce":1285694746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6807","nonce":1285694746}]},"public_addr":"172.21.15.148:6803/1285694746","cluster_addr":"172.21.15.148:6805/1285694746","heartbeat_back_addr":"172.21.15.148:6809/1285694746","heartbeat_front_addr":"172.21.15.148:6807/1285694746","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6811","nonce":3763364013}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6813","nonce":3763364013}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6817","nonce":3763364013}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6815","nonce":3763364013}]},"public_addr":"172.21.15.148:6811/3763364013","cluster_addr":"172.21.15.148:6813/3763364013","heartbeat_back_addr":"172.21.15.148:6817/3763364013","heartbeat_front_addr":"172.21.15.148:6815/3763364013","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6819","nonce":916957575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6821","nonce":916957575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6825","nonce":916957575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6823","nonce":916957575}]},"public_addr":"172.21.15.148:6819/916957575","cluster_addr":"172.21.15.148:6821/916957575","heartbeat_back_addr":"172.21.15.148:6825/916957575","heartbeat_front_addr":"172.21.15.148:6823/916957575","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6827","nonce":2267330328}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6829","nonce":2267330328}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6833","nonce":2267330328}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6831","nonce":2267330328}]},"public_addr":"172.21.15.148:6827/2267330328","cluster_addr":"172.21.15.148:6829/2267330328","heartbeat_back_addr":"172.21.15.148:6833/2267330328","heartbeat_front_addr":"172.21.15.148:6831/2267330328","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6801","nonce":1363408518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6803","nonce":1363408518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6807","nonce":1363408518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6805","nonce":1363408518}]},"public_addr":"172.21.15.195:6801/1363408518","cluster_addr":"172.21.15.195:6803/1363408518","heartbeat_back_addr":"172.21.15.195:6807/1363408518","heartbeat_front_addr":"172.21.15.195:6805/1363408518","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:08:09.499 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:09 smithi148 bash[21362]: cluster 2024-03-07T03:08:07.275790+0000 mgr.y (mgr.14152) 315 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-07T03:08:09.499 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:09 smithi148 bash[31149]: cluster 2024-03-07T03:08:07.275790+0000 mgr.y (mgr.14152) 315 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-07T03:08:09.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:09 smithi195 bash[23221]: cluster 2024-03-07T03:08:07.275790+0000 mgr.y (mgr.14152) 315 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-03-07T03:08:10.095 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-07T03:03:43.780289+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-03-07T03:08:10.095 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-07T03:08:10.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:10 smithi148 bash[21362]: audit 2024-03-07T03:08:09.490486+0000 mon.a (mon.0) 641 : audit 0 from='client.? 172.21.15.148:0/511384059' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:08:10.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:10 smithi148 bash[31149]: audit 2024-03-07T03:08:09.490486+0000 mon.a (mon.0) 641 : audit 0 from='client.? 172.21.15.148:0/511384059' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:08:10.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:10 smithi195 bash[23221]: audit 2024-03-07T03:08:09.490486+0000 mon.a (mon.0) 641 : audit 0 from='client.? 172.21.15.148:0/511384059' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:08:11.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:11 smithi148 bash[21362]: cluster 2024-03-07T03:08:09.276206+0000 mgr.y (mgr.14152) 316 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:08:11.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:11 smithi148 bash[31149]: cluster 2024-03-07T03:08:09.276206+0000 mgr.y (mgr.14152) 316 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:08:11.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:11 smithi195 bash[23221]: cluster 2024-03-07T03:08:09.276206+0000 mgr.y (mgr.14152) 316 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-07T03:08:12.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:12 smithi148 bash[21362]: cluster 2024-03-07T03:08:11.276624+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:12.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:12 smithi148 bash[31149]: cluster 2024-03-07T03:08:11.276624+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:12.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:12 smithi195 bash[23221]: cluster 2024-03-07T03:08:11.276624+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:14.807 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:08:15.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:15 smithi148 bash[21362]: cluster 2024-03-07T03:08:13.277163+0000 mgr.y (mgr.14152) 318 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:15.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:15 smithi148 bash[31149]: cluster 2024-03-07T03:08:13.277163+0000 mgr.y (mgr.14152) 318 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:15.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:15 smithi195 bash[23221]: cluster 2024-03-07T03:08:13.277163+0000 mgr.y (mgr.14152) 318 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:17.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:17 smithi148 bash[21362]: cluster 2024-03-07T03:08:15.277764+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:17.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:17 smithi148 bash[31149]: cluster 2024-03-07T03:08:15.277764+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:17.351 INFO:teuthology.orchestra.run.smithi148.stdout:pg_num: 1 2024-03-07T03:08:17.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:17 smithi195 bash[23221]: cluster 2024-03-07T03:08:15.277764+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:17.984 INFO:tasks.cephadm:Adding prometheus.a on smithi195 2024-03-07T03:08:17.984 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply prometheus '1;smithi195=a' 2024-03-07T03:08:18.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:18 smithi148 bash[21362]: audit 2024-03-07T03:08:17.359482+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.148:0/3453949465' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-07T03:08:18.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:18 smithi148 bash[31149]: audit 2024-03-07T03:08:17.359482+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.148:0/3453949465' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-07T03:08:18.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:18 smithi195 bash[23221]: audit 2024-03-07T03:08:17.359482+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.148:0/3453949465' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-07T03:08:19.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:19 smithi148 bash[21362]: cluster 2024-03-07T03:08:17.278367+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:19.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:19 smithi148 bash[31149]: cluster 2024-03-07T03:08:17.278367+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:19.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:19 smithi195 bash[23221]: cluster 2024-03-07T03:08:17.278367+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:21.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:21 smithi148 bash[21362]: cluster 2024-03-07T03:08:19.279003+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:21.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:21 smithi148 bash[31149]: cluster 2024-03-07T03:08:19.279003+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:21.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:21 smithi195 bash[23221]: cluster 2024-03-07T03:08:19.279003+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:22.687 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:08:23.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:23 smithi148 bash[21362]: cluster 2024-03-07T03:08:21.279515+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:23.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:23 smithi148 bash[31149]: cluster 2024-03-07T03:08:21.279515+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:23.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:23 smithi195 bash[23221]: cluster 2024-03-07T03:08:21.279515+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:25.539 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled prometheus update... 2024-03-07T03:08:25.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:25 smithi148 bash[21362]: cluster 2024-03-07T03:08:23.280107+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:25.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:25 smithi148 bash[31149]: cluster 2024-03-07T03:08:23.280107+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:25.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:25 smithi195 bash[23221]: cluster 2024-03-07T03:08:23.280107+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:26.197 DEBUG:teuthology.orchestra.run.smithi195:prometheus.a> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service 2024-03-07T03:08:26.199 INFO:tasks.cephadm:Adding node-exporter.a on smithi148 2024-03-07T03:08:26.199 INFO:tasks.cephadm:Adding node-exporter.b on smithi195 2024-03-07T03:08:26.199 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply node-exporter '2;smithi148=a;smithi195=b' 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: cluster 2024-03-07T03:08:25.280668+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: audit 2024-03-07T03:08:25.535046+0000 mgr.y (mgr.14152) 325 : audit 0 from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: cephadm 2024-03-07T03:08:25.538507+0000 mgr.y (mgr.14152) 326 : cephadm 1 Saving service prometheus spec with placement smithi195=a;count:1 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: audit 2024-03-07T03:08:25.545858+0000 mon.a (mon.0) 642 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: audit 2024-03-07T03:08:25.550605+0000 mon.a (mon.0) 643 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: audit 2024-03-07T03:08:25.554485+0000 mon.a (mon.0) 644 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: audit 2024-03-07T03:08:25.556361+0000 mon.a (mon.0) 645 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:26.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: audit 2024-03-07T03:08:25.567015+0000 mon.a (mon.0) 646 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:26.797 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[21362]: cephadm 2024-03-07T03:08:25.858152+0000 mgr.y (mgr.14152) 327 : cephadm 1 Deploying daemon prometheus.a on smithi195 2024-03-07T03:08:26.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: cluster 2024-03-07T03:08:25.280668+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:26.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: audit 2024-03-07T03:08:25.535046+0000 mgr.y (mgr.14152) 325 : audit 0 from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:26.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: cephadm 2024-03-07T03:08:25.538507+0000 mgr.y (mgr.14152) 326 : cephadm 1 Saving service prometheus spec with placement smithi195=a;count:1 2024-03-07T03:08:26.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: audit 2024-03-07T03:08:25.545858+0000 mon.a (mon.0) 642 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:26.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: audit 2024-03-07T03:08:25.550605+0000 mon.a (mon.0) 643 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:26.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: audit 2024-03-07T03:08:25.554485+0000 mon.a (mon.0) 644 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:26.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: audit 2024-03-07T03:08:25.556361+0000 mon.a (mon.0) 645 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:26.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: audit 2024-03-07T03:08:25.567015+0000 mon.a (mon.0) 646 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:26.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:26 smithi148 bash[31149]: cephadm 2024-03-07T03:08:25.858152+0000 mgr.y (mgr.14152) 327 : cephadm 1 Deploying daemon prometheus.a on smithi195 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: cluster 2024-03-07T03:08:25.280668+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: audit 2024-03-07T03:08:25.535046+0000 mgr.y (mgr.14152) 325 : audit 0 from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: cephadm 2024-03-07T03:08:25.538507+0000 mgr.y (mgr.14152) 326 : cephadm 1 Saving service prometheus spec with placement smithi195=a;count:1 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: audit 2024-03-07T03:08:25.545858+0000 mon.a (mon.0) 642 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: audit 2024-03-07T03:08:25.550605+0000 mon.a (mon.0) 643 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: audit 2024-03-07T03:08:25.554485+0000 mon.a (mon.0) 644 : audit 0 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: audit 2024-03-07T03:08:25.556361+0000 mon.a (mon.0) 645 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: audit 2024-03-07T03:08:25.567015+0000 mon.a (mon.0) 646 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:26.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:26 smithi195 bash[23221]: cephadm 2024-03-07T03:08:25.858152+0000 mgr.y (mgr.14152) 327 : cephadm 1 Deploying daemon prometheus.a on smithi195 2024-03-07T03:08:29.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:29 smithi148 bash[21362]: cluster 2024-03-07T03:08:27.281342+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:29.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:29 smithi148 bash[31149]: cluster 2024-03-07T03:08:27.281342+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:29.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:29 smithi195 bash[23221]: cluster 2024-03-07T03:08:27.281342+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:30.914 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:08:31.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:31 smithi195 bash[23221]: cluster 2024-03-07T03:08:29.281927+0000 mgr.y (mgr.14152) 329 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:31.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:31 smithi148 bash[21362]: cluster 2024-03-07T03:08:29.281927+0000 mgr.y (mgr.14152) 329 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:31.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:31 smithi148 bash[31149]: cluster 2024-03-07T03:08:29.281927+0000 mgr.y (mgr.14152) 329 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:33.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:33 smithi148 bash[21362]: cluster 2024-03-07T03:08:31.282451+0000 mgr.y (mgr.14152) 330 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:33.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:33 smithi148 bash[31149]: cluster 2024-03-07T03:08:31.282451+0000 mgr.y (mgr.14152) 330 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:33.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:33 smithi195 bash[23221]: cluster 2024-03-07T03:08:31.282451+0000 mgr.y (mgr.14152) 330 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:35.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:35 smithi148 bash[21362]: cluster 2024-03-07T03:08:33.283019+0000 mgr.y (mgr.14152) 331 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:35.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:35 smithi148 bash[31149]: cluster 2024-03-07T03:08:33.283019+0000 mgr.y (mgr.14152) 331 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:35.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:35 smithi195 bash[23221]: cluster 2024-03-07T03:08:33.283019+0000 mgr.y (mgr.14152) 331 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:37.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:37 smithi195 bash[23221]: cluster 2024-03-07T03:08:35.283635+0000 mgr.y (mgr.14152) 332 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:37.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:37 smithi148 bash[21362]: cluster 2024-03-07T03:08:35.283635+0000 mgr.y (mgr.14152) 332 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:37.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:37 smithi148 bash[31149]: cluster 2024-03-07T03:08:35.283635+0000 mgr.y (mgr.14152) 332 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:38.325 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled node-exporter update... 2024-03-07T03:08:39.346 DEBUG:teuthology.orchestra.run.smithi148:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@node-exporter.a.service 2024-03-07T03:08:39.349 DEBUG:teuthology.orchestra.run.smithi195:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@node-exporter.b.service 2024-03-07T03:08:39.351 INFO:tasks.cephadm:Adding alertmanager.a on smithi148 2024-03-07T03:08:39.351 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply alertmanager '1;smithi148=a' 2024-03-07T03:08:39.511 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.511 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.511 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:39 smithi195 bash[23221]: cluster 2024-03-07T03:08:37.284252+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:39.512 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:39 smithi195 bash[23221]: audit 2024-03-07T03:08:38.332934+0000 mon.a (mon.0) 647 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:39.512 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.512 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.512 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.513 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:39 smithi148 bash[21362]: cluster 2024-03-07T03:08:37.284252+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:39.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:39 smithi148 bash[21362]: audit 2024-03-07T03:08:38.332934+0000 mon.a (mon.0) 647 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:39.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:39 smithi148 bash[31149]: cluster 2024-03-07T03:08:37.284252+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:39.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:39 smithi148 bash[31149]: audit 2024-03-07T03:08:38.332934+0000 mon.a (mon.0) 647 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:39.644 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.644 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.644 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.644 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.802 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.802 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.803 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.803 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.803 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:39.804 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: Started Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:08:39.804 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:39 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:40.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21362]: audit 2024-03-07T03:08:38.321897+0000 mgr.y (mgr.14152) 334 : audit 0 from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi148=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:40.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21362]: cephadm 2024-03-07T03:08:38.325567+0000 mgr.y (mgr.14152) 335 : cephadm 1 Saving service node-exporter spec with placement smithi148=a;smithi195=b;count:2 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21362]: audit 2024-03-07T03:08:39.808640+0000 mon.a (mon.0) 648 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21362]: audit 2024-03-07T03:08:39.818831+0000 mon.a (mon.0) 649 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21362]: audit 2024-03-07T03:08:39.867508+0000 mon.a (mon.0) 650 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21362]: audit 2024-03-07T03:08:39.875253+0000 mon.a (mon.0) 651 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[31149]: audit 2024-03-07T03:08:38.321897+0000 mgr.y (mgr.14152) 334 : audit 0 from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi148=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[31149]: cephadm 2024-03-07T03:08:38.325567+0000 mgr.y (mgr.14152) 335 : cephadm 1 Saving service node-exporter spec with placement smithi148=a;smithi195=b;count:2 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[31149]: audit 2024-03-07T03:08:39.808640+0000 mon.a (mon.0) 648 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[31149]: audit 2024-03-07T03:08:39.818831+0000 mon.a (mon.0) 649 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[31149]: audit 2024-03-07T03:08:39.867508+0000 mon.a (mon.0) 650 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[31149]: audit 2024-03-07T03:08:39.875253+0000 mon.a (mon.0) 651 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-07T03:08:40.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[23221]: audit 2024-03-07T03:08:38.321897+0000 mgr.y (mgr.14152) 334 : audit 0 from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi148=a;smithi195=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:40.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[23221]: cephadm 2024-03-07T03:08:38.325567+0000 mgr.y (mgr.14152) 335 : cephadm 1 Saving service node-exporter spec with placement smithi148=a;smithi195=b;count:2 2024-03-07T03:08:40.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[23221]: audit 2024-03-07T03:08:39.808640+0000 mon.a (mon.0) 648 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[23221]: audit 2024-03-07T03:08:39.818831+0000 mon.a (mon.0) 649 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[23221]: audit 2024-03-07T03:08:39.867508+0000 mon.a (mon.0) 650 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' 2024-03-07T03:08:40.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[23221]: audit 2024-03-07T03:08:39.875253+0000 mon.a (mon.0) 651 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-07T03:08:41.052 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[24174]: ignoring --setuser ceph since I am not root 2024-03-07T03:08:41.052 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[24174]: ignoring --setgroup ceph since I am not root 2024-03-07T03:08:41.052 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[24174]: debug 2024-03-07T03:08:40.921+0000 7f602b4cf700 1 -- 172.21.15.195:0/2513379723 <== mon.0 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d7ebc7d1e0 con 0x55d7ec9fe400 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.878Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.878Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.878Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi195 (none))" 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.878Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.878Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.880Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.880Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.882Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.882Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.884Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:08:41.053 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.884Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.043µs 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.884Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.885Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.885Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.763µs wal_replay_duration=339.796µs wbl_replay_duration=123ns total_replay_duration=381.813µs 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.886Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.886Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.886Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.907Z caller=refresh.go:80 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=ceph-exporter\": read tcp 172.21.15.195:44396->172.21.15.148:8765: read: connection reset by peer" 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.907Z caller=refresh.go:80 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=mgr-prometheus\": read tcp 172.21.15.195:44398->172.21.15.148:8765: read: connection reset by peer" 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.907Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.315517ms db_storage=1.385µs remote_storage=1.263µs web_handler=447ns query_engine=589ns scrape=283.379µs scrape_sd=97.35µs notify=1.059µs notify_sd=2.769µs rules=20.693251ms tracing=8.33µs 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.907Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:08:41.054 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:40 smithi195 bash[47245]: ts=2024-03-07T03:08:40.907Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:08:41.201 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21615]: ignoring --setuser ceph since I am not root 2024-03-07T03:08:41.201 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:40 smithi148 bash[21615]: ignoring --setgroup ceph since I am not root 2024-03-07T03:08:41.201 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:41 smithi148 bash[21615]: debug 2024-03-07T03:08:41.077+0000 7f37e6cbc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:08:41.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:41 smithi195 bash[23221]: cluster 2024-03-07T03:08:39.284790+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:41.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:41 smithi195 bash[23221]: audit 2024-03-07T03:08:40.878887+0000 mon.a (mon.0) 652 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-07T03:08:41.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:41 smithi195 bash[23221]: cluster 2024-03-07T03:08:40.878949+0000 mon.a (mon.0) 653 : cluster 0 mgrmap e16: y(active, since 8m), standbys: x 2024-03-07T03:08:41.319 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:41 smithi195 bash[24174]: debug 2024-03-07T03:08:41.053+0000 7f6035213200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:08:41.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:41 smithi148 bash[21362]: cluster 2024-03-07T03:08:39.284790+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:41.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:41 smithi148 bash[21362]: audit 2024-03-07T03:08:40.878887+0000 mon.a (mon.0) 652 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-07T03:08:41.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:41 smithi148 bash[21362]: cluster 2024-03-07T03:08:40.878949+0000 mon.a (mon.0) 653 : cluster 0 mgrmap e16: y(active, since 8m), standbys: x 2024-03-07T03:08:41.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:41 smithi148 bash[31149]: cluster 2024-03-07T03:08:39.284790+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:41.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:41 smithi148 bash[31149]: audit 2024-03-07T03:08:40.878887+0000 mon.a (mon.0) 652 : audit 1 from='mgr.14152 172.21.15.148:0/3963212827' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-07T03:08:41.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:41 smithi148 bash[31149]: cluster 2024-03-07T03:08:40.878949+0000 mon.a (mon.0) 653 : cluster 0 mgrmap e16: y(active, since 8m), standbys: x 2024-03-07T03:08:42.935 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:42 smithi195 bash[24174]: debug 2024-03-07T03:08:42.597+0000 7f6035213200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:08:42.936 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:42 smithi195 bash[24174]: debug 2024-03-07T03:08:42.689+0000 7f6035213200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:08:42.936 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:42 smithi195 bash[24174]: debug 2024-03-07T03:08:42.937+0000 7f6035213200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:08:42.977 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:42 smithi148 bash[21615]: debug 2024-03-07T03:08:42.637+0000 7f37e6cbc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:08:42.978 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:42 smithi148 bash[21615]: debug 2024-03-07T03:08:42.733+0000 7f37e6cbc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:08:42.978 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:42 smithi148 bash[21615]: debug 2024-03-07T03:08:42.981+0000 7f37e6cbc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:08:44.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:43 smithi148 bash[21615]: debug 2024-03-07T03:08:43.769+0000 7f37e6cbc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:08:44.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:43 smithi148 bash[21615]: debug 2024-03-07T03:08:43.877+0000 7f37e6cbc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:08:44.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:43 smithi148 bash[21615]: debug 2024-03-07T03:08:43.981+0000 7f37e6cbc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:08:44.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:43 smithi195 bash[24174]: debug 2024-03-07T03:08:43.729+0000 7f6035213200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:08:44.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:43 smithi195 bash[24174]: debug 2024-03-07T03:08:43.837+0000 7f6035213200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:08:44.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:43 smithi195 bash[24174]: debug 2024-03-07T03:08:43.941+0000 7f6035213200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:08:44.568 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:44 smithi195 bash[24174]: debug 2024-03-07T03:08:44.293+0000 7f6035213200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:08:44.568 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:44 smithi195 bash[24174]: debug 2024-03-07T03:08:44.405+0000 7f6035213200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:08:44.569 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:44 smithi195 bash[24174]: debug 2024-03-07T03:08:44.505+0000 7f6035213200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:08:44.795 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:44 smithi148 bash[21615]: debug 2024-03-07T03:08:44.337+0000 7f37e6cbc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:08:44.795 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:44 smithi148 bash[21615]: debug 2024-03-07T03:08:44.449+0000 7f37e6cbc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:08:44.795 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:44 smithi148 bash[21615]: debug 2024-03-07T03:08:44.553+0000 7f37e6cbc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:08:45.089 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:08:45.110 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:44 smithi195 bash[24174]: debug 2024-03-07T03:08:44.857+0000 7f6035213200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:08:45.110 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:45 smithi195 bash[24174]: debug 2024-03-07T03:08:45.097+0000 7f6035213200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:08:45.295 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:44 smithi148 bash[21615]: debug 2024-03-07T03:08:44.909+0000 7f37e6cbc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:08:45.295 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:45 smithi148 bash[21615]: debug 2024-03-07T03:08:45.149+0000 7f37e6cbc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:08:45.462 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:45 smithi195 bash[24174]: debug 2024-03-07T03:08:45.357+0000 7f6035213200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:08:45.735 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:45 smithi195 bash[24174]: debug 2024-03-07T03:08:45.465+0000 7f6035213200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:08:45.794 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:45 smithi148 bash[21615]: debug 2024-03-07T03:08:45.409+0000 7f37e6cbc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:08:45.794 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:45 smithi148 bash[21615]: debug 2024-03-07T03:08:45.521+0000 7f37e6cbc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:08:46.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:45 smithi148 bash[21615]: debug 2024-03-07T03:08:45.797+0000 7f37e6cbc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:08:46.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:46 smithi148 bash[21615]: debug 2024-03-07T03:08:46.041+0000 7f37e6cbc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:08:46.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:45 smithi195 bash[24174]: debug 2024-03-07T03:08:45.737+0000 7f6035213200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:08:46.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:45 smithi195 bash[24174]: debug 2024-03-07T03:08:45.977+0000 7f6035213200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:08:46.510 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:46 smithi195 bash[24174]: debug 2024-03-07T03:08:46.393+0000 7f6035213200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:08:46.795 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:46 smithi148 bash[21615]: debug 2024-03-07T03:08:46.449+0000 7f37e6cbc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:08:46.795 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:46 smithi148 bash[21615]: debug 2024-03-07T03:08:46.565+0000 7f37e6cbc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:08:46.818 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:46 smithi195 bash[24174]: debug 2024-03-07T03:08:46.513+0000 7f6035213200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:08:46.818 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:46 smithi195 bash[24174]: debug 2024-03-07T03:08:46.757+0000 7f6035213200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:08:47.295 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:46 smithi148 bash[21615]: debug 2024-03-07T03:08:46.813+0000 7f37e6cbc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:08:48.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:47 smithi195 bash[24174]: debug 2024-03-07T03:08:47.764+0000 7f6035213200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:08:48.068 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:47 smithi195 bash[24174]: debug 2024-03-07T03:08:47.880+0000 7f6035213200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:08:48.226 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:47 smithi148 bash[21615]: debug 2024-03-07T03:08:47.817+0000 7f37e6cbc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:08:48.227 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:47 smithi148 bash[21615]: debug 2024-03-07T03:08:47.933+0000 7f37e6cbc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:08:48.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:48 smithi148 bash[21615]: debug 2024-03-07T03:08:48.229+0000 7f37e6cbc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:08:48.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:48 smithi148 bash[21615]: debug 2024-03-07T03:08:48.345+0000 7f37e6cbc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:08:48.568 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: debug 2024-03-07T03:08:48.176+0000 7f6035213200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:08:48.568 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: debug 2024-03-07T03:08:48.292+0000 7f6035213200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:08:48.988 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: debug 2024-03-07T03:08:48.736+0000 7f6035213200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:08:48.989 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: debug 2024-03-07T03:08:48.856+0000 7f6035213200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:08:48.989 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: [07/Mar/2024:03:08:48] ENGINE Bus STARTING 2024-03-07T03:08:48.989 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: CherryPy Checker: 2024-03-07T03:08:49.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:48 smithi148 bash[21615]: debug 2024-03-07T03:08:48.797+0000 7f37e6cbc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:08:49.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:48 smithi148 bash[21615]: debug 2024-03-07T03:08:48.921+0000 7f37e6cbc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:08:49.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21615]: [07/Mar/2024:03:08:49] ENGINE Bus STARTING 2024-03-07T03:08:49.318 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: The Application mounted at '' has an empty config. 2024-03-07T03:08:49.319 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: [07/Mar/2024:03:08:48] ENGINE Serving on http://:::9283 2024-03-07T03:08:49.319 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:48 smithi195 bash[24174]: [07/Mar/2024:03:08:48] ENGINE Bus STARTED 2024-03-07T03:08:49.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: cluster 2024-03-07T03:08:48.886102+0000 mon.a (mon.0) 654 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:08:49.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: cluster 2024-03-07T03:08:48.886294+0000 mon.a (mon.0) 655 : cluster 0 Standby manager daemon x started 2024-03-07T03:08:49.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.889270+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:08:49.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.890289+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.892856+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.894782+0000 mon.b (mon.1) 29 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: cluster 2024-03-07T03:08:48.946702+0000 mon.a (mon.0) 656 : cluster 1 Active manager daemon y restarted 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: cluster 2024-03-07T03:08:48.947292+0000 mon.a (mon.0) 657 : cluster 1 Activating manager daemon y 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: cluster 2024-03-07T03:08:48.956317+0000 mon.a (mon.0) 658 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: cluster 2024-03-07T03:08:48.956634+0000 mon.a (mon.0) 659 : cluster 0 mgrmap e17: y(active, starting, since 0.00953353s), standbys: x 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.961539+0000 mon.b (mon.1) 30 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.961766+0000 mon.b (mon.1) 31 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.961989+0000 mon.b (mon.1) 32 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.963732+0000 mon.b (mon.1) 33 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.963999+0000 mon.b (mon.1) 34 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.964768+0000 mon.b (mon.1) 35 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:08:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.964980+0000 mon.b (mon.1) 36 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.965778+0000 mon.b (mon.1) 37 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.966053+0000 mon.b (mon.1) 38 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.966357+0000 mon.b (mon.1) 39 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.966657+0000 mon.b (mon.1) 40 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.966957+0000 mon.b (mon.1) 41 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.967253+0000 mon.b (mon.1) 42 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.967564+0000 mon.b (mon.1) 43 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.967891+0000 mon.b (mon.1) 44 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:48.968201+0000 mon.b (mon.1) 45 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: cluster 2024-03-07T03:08:48.995641+0000 mon.a (mon.0) 660 : cluster 1 Manager daemon y is now available 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.017702+0000 mon.b (mon.1) 46 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.019978+0000 mon.b (mon.1) 47 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.039071+0000 mon.a (mon.0) 661 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:49.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.042101+0000 mon.b (mon.1) 48 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:49.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.052860+0000 mon.b (mon.1) 49 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:08:49.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.054747+0000 mon.a (mon.0) 662 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:08:49.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.065839+0000 mon.b (mon.1) 50 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:08:49.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:49 smithi195 bash[23221]: audit 2024-03-07T03:08:49.067790+0000 mon.a (mon.0) 663 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:08:49.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21615]: CherryPy Checker: 2024-03-07T03:08:49.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21615]: The Application mounted at '' has an empty config. 2024-03-07T03:08:49.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21615]: [07/Mar/2024:03:08:49] ENGINE Serving on http://:::9283 2024-03-07T03:08:49.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21615]: [07/Mar/2024:03:08:49] ENGINE Bus STARTED 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: cluster 2024-03-07T03:08:48.886102+0000 mon.a (mon.0) 654 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: cluster 2024-03-07T03:08:48.886294+0000 mon.a (mon.0) 655 : cluster 0 Standby manager daemon x started 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.889270+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.890289+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.892856+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.894782+0000 mon.b (mon.1) 29 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: cluster 2024-03-07T03:08:48.946702+0000 mon.a (mon.0) 656 : cluster 1 Active manager daemon y restarted 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: cluster 2024-03-07T03:08:48.947292+0000 mon.a (mon.0) 657 : cluster 1 Activating manager daemon y 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: cluster 2024-03-07T03:08:48.956317+0000 mon.a (mon.0) 658 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: cluster 2024-03-07T03:08:48.956634+0000 mon.a (mon.0) 659 : cluster 0 mgrmap e17: y(active, starting, since 0.00953353s), standbys: x 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.961539+0000 mon.b (mon.1) 30 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:08:49.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.961766+0000 mon.b (mon.1) 31 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.961989+0000 mon.b (mon.1) 32 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.963732+0000 mon.b (mon.1) 33 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.963999+0000 mon.b (mon.1) 34 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.964768+0000 mon.b (mon.1) 35 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.964980+0000 mon.b (mon.1) 36 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.965778+0000 mon.b (mon.1) 37 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.966053+0000 mon.b (mon.1) 38 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.966357+0000 mon.b (mon.1) 39 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.966657+0000 mon.b (mon.1) 40 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.966957+0000 mon.b (mon.1) 41 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.967253+0000 mon.b (mon.1) 42 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.967564+0000 mon.b (mon.1) 43 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.967891+0000 mon.b (mon.1) 44 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:48.968201+0000 mon.b (mon.1) 45 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: cluster 2024-03-07T03:08:48.995641+0000 mon.a (mon.0) 660 : cluster 1 Manager daemon y is now available 2024-03-07T03:08:49.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.017702+0000 mon.b (mon.1) 46 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.019978+0000 mon.b (mon.1) 47 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.039071+0000 mon.a (mon.0) 661 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.042101+0000 mon.b (mon.1) 48 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.052860+0000 mon.b (mon.1) 49 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.054747+0000 mon.a (mon.0) 662 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.065839+0000 mon.b (mon.1) 50 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[21362]: audit 2024-03-07T03:08:49.067790+0000 mon.a (mon.0) 663 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:08:49.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: cluster 2024-03-07T03:08:48.886102+0000 mon.a (mon.0) 654 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: cluster 2024-03-07T03:08:48.886294+0000 mon.a (mon.0) 655 : cluster 0 Standby manager daemon x started 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.889270+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.890289+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.892856+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.894782+0000 mon.b (mon.1) 29 : audit 0 from='mgr.? 172.21.15.195:0/142157658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: cluster 2024-03-07T03:08:48.946702+0000 mon.a (mon.0) 656 : cluster 1 Active manager daemon y restarted 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: cluster 2024-03-07T03:08:48.947292+0000 mon.a (mon.0) 657 : cluster 1 Activating manager daemon y 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: cluster 2024-03-07T03:08:48.956317+0000 mon.a (mon.0) 658 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: cluster 2024-03-07T03:08:48.956634+0000 mon.a (mon.0) 659 : cluster 0 mgrmap e17: y(active, starting, since 0.00953353s), standbys: x 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.961539+0000 mon.b (mon.1) 30 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.961766+0000 mon.b (mon.1) 31 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.961989+0000 mon.b (mon.1) 32 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.963732+0000 mon.b (mon.1) 33 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.963999+0000 mon.b (mon.1) 34 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:08:49.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.964768+0000 mon.b (mon.1) 35 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.964980+0000 mon.b (mon.1) 36 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.965778+0000 mon.b (mon.1) 37 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.966053+0000 mon.b (mon.1) 38 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.966357+0000 mon.b (mon.1) 39 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.966657+0000 mon.b (mon.1) 40 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.966957+0000 mon.b (mon.1) 41 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.967253+0000 mon.b (mon.1) 42 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.967564+0000 mon.b (mon.1) 43 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.967891+0000 mon.b (mon.1) 44 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:48.968201+0000 mon.b (mon.1) 45 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: cluster 2024-03-07T03:08:48.995641+0000 mon.a (mon.0) 660 : cluster 1 Manager daemon y is now available 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.017702+0000 mon.b (mon.1) 46 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.019978+0000 mon.b (mon.1) 47 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.039071+0000 mon.a (mon.0) 661 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.042101+0000 mon.b (mon.1) 48 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:08:49.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.052860+0000 mon.b (mon.1) 49 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:08:49.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.054747+0000 mon.a (mon.0) 662 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:08:49.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.065839+0000 mon.b (mon.1) 50 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:08:49.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:49 smithi148 bash[31149]: audit 2024-03-07T03:08:49.067790+0000 mon.a (mon.0) 663 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:08:49.986 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled alertmanager update... 2024-03-07T03:08:50.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:50 smithi195 bash[23221]: cephadm 2024-03-07T03:08:49.443146+0000 mgr.y (mgr.24287) 1 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Bus STARTING 2024-03-07T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:50 smithi195 bash[23221]: cephadm 2024-03-07T03:08:49.546798+0000 mgr.y (mgr.24287) 2 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:50 smithi195 bash[23221]: cephadm 2024-03-07T03:08:49.670452+0000 mgr.y (mgr.24287) 3 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:50 smithi195 bash[23221]: cephadm 2024-03-07T03:08:49.670545+0000 mgr.y (mgr.24287) 4 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Bus STARTED 2024-03-07T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:50 smithi195 bash[23221]: cluster 2024-03-07T03:08:49.970981+0000 mon.a (mon.0) 664 : cluster 0 mgrmap e18: y(active, since 1.02385s), standbys: x 2024-03-07T03:08:50.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:50 smithi195 bash[23221]: audit 2024-03-07T03:08:49.991560+0000 mon.a (mon.0) 665 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:50.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[21362]: cephadm 2024-03-07T03:08:49.443146+0000 mgr.y (mgr.24287) 1 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Bus STARTING 2024-03-07T03:08:50.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[21362]: cephadm 2024-03-07T03:08:49.546798+0000 mgr.y (mgr.24287) 2 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:08:50.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[21362]: cephadm 2024-03-07T03:08:49.670452+0000 mgr.y (mgr.24287) 3 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:08:50.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[21362]: cephadm 2024-03-07T03:08:49.670545+0000 mgr.y (mgr.24287) 4 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Bus STARTED 2024-03-07T03:08:50.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[21362]: cluster 2024-03-07T03:08:49.970981+0000 mon.a (mon.0) 664 : cluster 0 mgrmap e18: y(active, since 1.02385s), standbys: x 2024-03-07T03:08:50.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[21362]: audit 2024-03-07T03:08:49.991560+0000 mon.a (mon.0) 665 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:50.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[31149]: cephadm 2024-03-07T03:08:49.443146+0000 mgr.y (mgr.24287) 1 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Bus STARTING 2024-03-07T03:08:50.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[31149]: cephadm 2024-03-07T03:08:49.546798+0000 mgr.y (mgr.24287) 2 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:08:50.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[31149]: cephadm 2024-03-07T03:08:49.670452+0000 mgr.y (mgr.24287) 3 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:08:50.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[31149]: cephadm 2024-03-07T03:08:49.670545+0000 mgr.y (mgr.24287) 4 : cephadm 1 [07/Mar/2024:03:08:49] ENGINE Bus STARTED 2024-03-07T03:08:50.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[31149]: cluster 2024-03-07T03:08:49.970981+0000 mon.a (mon.0) 664 : cluster 0 mgrmap e18: y(active, since 1.02385s), standbys: x 2024-03-07T03:08:50.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:50 smithi148 bash[31149]: audit 2024-03-07T03:08:49.991560+0000 mon.a (mon.0) 665 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:50.796 DEBUG:teuthology.orchestra.run.smithi148:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@alertmanager.a.service 2024-03-07T03:08:50.799 INFO:tasks.cephadm:Adding grafana.a on smithi195 2024-03-07T03:08:50.799 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph orch apply grafana '1;smithi195=a' 2024-03-07T03:08:51.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:51 smithi148 bash[21362]: audit 2024-03-07T03:08:49.981572+0000 mgr.y (mgr.24287) 5 : audit 0 from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi148=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:51.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:51 smithi148 bash[21362]: cephadm 2024-03-07T03:08:49.985240+0000 mgr.y (mgr.24287) 6 : cephadm 1 Saving service alertmanager spec with placement smithi148=a;count:1 2024-03-07T03:08:51.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:51 smithi148 bash[21362]: cluster 2024-03-07T03:08:49.989449+0000 mgr.y (mgr.24287) 7 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:51.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:51 smithi148 bash[31149]: audit 2024-03-07T03:08:49.981572+0000 mgr.y (mgr.24287) 5 : audit 0 from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi148=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:51.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:51 smithi148 bash[31149]: cephadm 2024-03-07T03:08:49.985240+0000 mgr.y (mgr.24287) 6 : cephadm 1 Saving service alertmanager spec with placement smithi148=a;count:1 2024-03-07T03:08:51.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:51 smithi148 bash[31149]: cluster 2024-03-07T03:08:49.989449+0000 mgr.y (mgr.24287) 7 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:51.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:51 smithi195 bash[23221]: audit 2024-03-07T03:08:49.981572+0000 mgr.y (mgr.24287) 5 : audit 0 from='client.24307 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi148=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:51.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:51 smithi195 bash[23221]: cephadm 2024-03-07T03:08:49.985240+0000 mgr.y (mgr.24287) 6 : cephadm 1 Saving service alertmanager spec with placement smithi148=a;count:1 2024-03-07T03:08:51.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:51 smithi195 bash[23221]: cluster 2024-03-07T03:08:49.989449+0000 mgr.y (mgr.24287) 7 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:52.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:52 smithi148 bash[21362]: cluster 2024-03-07T03:08:50.966555+0000 mgr.y (mgr.24287) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:52.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:52 smithi148 bash[21362]: cluster 2024-03-07T03:08:51.212034+0000 mon.a (mon.0) 666 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-03-07T03:08:52.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:52 smithi148 bash[31149]: cluster 2024-03-07T03:08:50.966555+0000 mgr.y (mgr.24287) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:52.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:52 smithi148 bash[31149]: cluster 2024-03-07T03:08:51.212034+0000 mon.a (mon.0) 666 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-03-07T03:08:52.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:52 smithi195 bash[23221]: cluster 2024-03-07T03:08:50.966555+0000 mgr.y (mgr.24287) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:52.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:52 smithi195 bash[23221]: cluster 2024-03-07T03:08:51.212034+0000 mon.a (mon.0) 666 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-03-07T03:08:54.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:54 smithi148 bash[21362]: cluster 2024-03-07T03:08:52.967111+0000 mgr.y (mgr.24287) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:54.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:54 smithi148 bash[21362]: cluster 2024-03-07T03:08:53.235204+0000 mon.a (mon.0) 667 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-03-07T03:08:54.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:54 smithi148 bash[31149]: cluster 2024-03-07T03:08:52.967111+0000 mgr.y (mgr.24287) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:54.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:54 smithi148 bash[31149]: cluster 2024-03-07T03:08:53.235204+0000 mon.a (mon.0) 667 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-03-07T03:08:54.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:54 smithi195 bash[23221]: cluster 2024-03-07T03:08:52.967111+0000 mgr.y (mgr.24287) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:54.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:54 smithi195 bash[23221]: cluster 2024-03-07T03:08:53.235204+0000 mon.a (mon.0) 667 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-03-07T03:08:55.326 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:08:55.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:55 smithi195 bash[23221]: audit 2024-03-07T03:08:54.588897+0000 mon.a (mon.0) 668 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:55.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:55 smithi195 bash[23221]: audit 2024-03-07T03:08:54.597209+0000 mon.a (mon.0) 669 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:55.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:55 smithi195 bash[23221]: audit 2024-03-07T03:08:54.827287+0000 mon.a (mon.0) 670 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:55.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:55 smithi195 bash[23221]: audit 2024-03-07T03:08:54.841368+0000 mon.a (mon.0) 671 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:55.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:55 smithi195 bash[23221]: audit 2024-03-07T03:08:55.404232+0000 mon.a (mon.0) 672 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:55.819 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:55 smithi195 bash[23221]: audit 2024-03-07T03:08:55.418552+0000 mon.a (mon.0) 673 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[31149]: audit 2024-03-07T03:08:54.588897+0000 mon.a (mon.0) 668 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[31149]: audit 2024-03-07T03:08:54.597209+0000 mon.a (mon.0) 669 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[31149]: audit 2024-03-07T03:08:54.827287+0000 mon.a (mon.0) 670 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[31149]: audit 2024-03-07T03:08:54.841368+0000 mon.a (mon.0) 671 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[31149]: audit 2024-03-07T03:08:55.404232+0000 mon.a (mon.0) 672 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[31149]: audit 2024-03-07T03:08:55.418552+0000 mon.a (mon.0) 673 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[21362]: audit 2024-03-07T03:08:54.588897+0000 mon.a (mon.0) 668 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[21362]: audit 2024-03-07T03:08:54.597209+0000 mon.a (mon.0) 669 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[21362]: audit 2024-03-07T03:08:54.827287+0000 mon.a (mon.0) 670 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[21362]: audit 2024-03-07T03:08:54.841368+0000 mon.a (mon.0) 671 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[21362]: audit 2024-03-07T03:08:55.404232+0000 mon.a (mon.0) 672 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:55 smithi148 bash[21362]: audit 2024-03-07T03:08:55.418552+0000 mon.a (mon.0) 673 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.913 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:56 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:56.913 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: cluster 2024-03-07T03:08:54.967526+0000 mgr.y (mgr.24287) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:56.913 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.663280+0000 mon.a (mon.0) 674 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.913 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.675624+0000 mon.a (mon.0) 675 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.913 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.678001+0000 mon.b (mon.1) 51 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.913 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.680272+0000 mon.a (mon.0) 676 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.913 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.681405+0000 mon.b (mon.1) 52 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.913 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.683607+0000 mon.a (mon.0) 677 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.684438+0000 mon.b (mon.1) 53 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.686592+0000 mon.a (mon.0) 678 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.687265+0000 mon.b (mon.1) 54 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.689460+0000 mon.a (mon.0) 679 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: cephadm 2024-03-07T03:08:55.691160+0000 mgr.y (mgr.24287) 11 : cephadm 1 Adjusting osd_memory_target on smithi195 to 4060M 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.701087+0000 mon.a (mon.0) 680 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.702060+0000 mon.b (mon.1) 55 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:55.704201+0000 mon.b (mon.1) 56 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: cephadm 2024-03-07T03:08:55.708466+0000 mgr.y (mgr.24287) 12 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: cephadm 2024-03-07T03:08:55.708931+0000 mgr.y (mgr.24287) 13 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:56.480100+0000 mon.a (mon.0) 681 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:56.486010+0000 mon.a (mon.0) 682 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:56.496091+0000 mon.a (mon.0) 683 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.914 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:56.507421+0000 mon.a (mon.0) 684 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[21362]: audit 2024-03-07T03:08:56.521312+0000 mon.a (mon.0) 685 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: cluster 2024-03-07T03:08:54.967526+0000 mgr.y (mgr.24287) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.663280+0000 mon.a (mon.0) 674 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.675624+0000 mon.a (mon.0) 675 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.678001+0000 mon.b (mon.1) 51 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.680272+0000 mon.a (mon.0) 676 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.681405+0000 mon.b (mon.1) 52 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.683607+0000 mon.a (mon.0) 677 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.684438+0000 mon.b (mon.1) 53 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.686592+0000 mon.a (mon.0) 678 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.687265+0000 mon.b (mon.1) 54 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.689460+0000 mon.a (mon.0) 679 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:56.915 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: cephadm 2024-03-07T03:08:55.691160+0000 mgr.y (mgr.24287) 11 : cephadm 1 Adjusting osd_memory_target on smithi195 to 4060M 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.701087+0000 mon.a (mon.0) 680 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.702060+0000 mon.b (mon.1) 55 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:55.704201+0000 mon.b (mon.1) 56 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: cephadm 2024-03-07T03:08:55.708466+0000 mgr.y (mgr.24287) 12 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: cephadm 2024-03-07T03:08:55.708931+0000 mgr.y (mgr.24287) 13 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:56.480100+0000 mon.a (mon.0) 681 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:56.486010+0000 mon.a (mon.0) 682 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:56.496091+0000 mon.a (mon.0) 683 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:56.507421+0000 mon.a (mon.0) 684 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:56.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:56 smithi148 bash[31149]: audit 2024-03-07T03:08:56.521312+0000 mon.a (mon.0) 685 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: cluster 2024-03-07T03:08:54.967526+0000 mgr.y (mgr.24287) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:08:57.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.663280+0000 mon.a (mon.0) 674 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.675624+0000 mon.a (mon.0) 675 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.678001+0000 mon.b (mon.1) 51 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.680272+0000 mon.a (mon.0) 676 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.681405+0000 mon.b (mon.1) 52 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.683607+0000 mon.a (mon.0) 677 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.684438+0000 mon.b (mon.1) 53 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.686592+0000 mon.a (mon.0) 678 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.687265+0000 mon.b (mon.1) 54 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.689460+0000 mon.a (mon.0) 679 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: cephadm 2024-03-07T03:08:55.691160+0000 mgr.y (mgr.24287) 11 : cephadm 1 Adjusting osd_memory_target on smithi195 to 4060M 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.701087+0000 mon.a (mon.0) 680 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.702060+0000 mon.b (mon.1) 55 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:55.704201+0000 mon.b (mon.1) 56 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:08:57.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: cephadm 2024-03-07T03:08:55.708466+0000 mgr.y (mgr.24287) 12 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:08:57.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: cephadm 2024-03-07T03:08:55.708931+0000 mgr.y (mgr.24287) 13 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:08:57.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:56.480100+0000 mon.a (mon.0) 681 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:56.486010+0000 mon.a (mon.0) 682 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:56.496091+0000 mon.a (mon.0) 683 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:56.507421+0000 mon.a (mon.0) 684 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:56 smithi195 bash[23221]: audit 2024-03-07T03:08:56.521312+0000 mon.a (mon.0) 685 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.201 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.202 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.202 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.202 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.203 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.203 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.203 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.482 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.483 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.483 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.483 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.484 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.484 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.484 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.484 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.484 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.484 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: Started Ceph node-exporter.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:08:57.485 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.485 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:08:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:57.795 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[55889]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-03-07T03:08:57.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: cephadm 2024-03-07T03:08:55.974034+0000 mgr.y (mgr.24287) 14 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:08:57.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: cephadm 2024-03-07T03:08:55.976679+0000 mgr.y (mgr.24287) 15 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:08:57.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: cephadm 2024-03-07T03:08:56.139441+0000 mgr.y (mgr.24287) 16 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:08:57.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: cephadm 2024-03-07T03:08:56.141697+0000 mgr.y (mgr.24287) 17 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:08:57.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: cephadm 2024-03-07T03:08:56.304208+0000 mgr.y (mgr.24287) 18 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:08:57.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: cephadm 2024-03-07T03:08:56.306383+0000 mgr.y (mgr.24287) 19 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: cephadm 2024-03-07T03:08:56.527873+0000 mgr.y (mgr.24287) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi148 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: audit 2024-03-07T03:08:57.484889+0000 mon.a (mon.0) 686 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: audit 2024-03-07T03:08:57.493316+0000 mon.a (mon.0) 687 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[21362]: audit 2024-03-07T03:08:57.550819+0000 mon.a (mon.0) 688 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: cephadm 2024-03-07T03:08:55.974034+0000 mgr.y (mgr.24287) 14 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: cephadm 2024-03-07T03:08:55.976679+0000 mgr.y (mgr.24287) 15 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: cephadm 2024-03-07T03:08:56.139441+0000 mgr.y (mgr.24287) 16 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: cephadm 2024-03-07T03:08:56.141697+0000 mgr.y (mgr.24287) 17 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: cephadm 2024-03-07T03:08:56.304208+0000 mgr.y (mgr.24287) 18 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: cephadm 2024-03-07T03:08:56.306383+0000 mgr.y (mgr.24287) 19 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: cephadm 2024-03-07T03:08:56.527873+0000 mgr.y (mgr.24287) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi148 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: audit 2024-03-07T03:08:57.484889+0000 mon.a (mon.0) 686 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: audit 2024-03-07T03:08:57.493316+0000 mon.a (mon.0) 687 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:57.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:57 smithi148 bash[31149]: audit 2024-03-07T03:08:57.550819+0000 mon.a (mon.0) 688 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.060 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:57 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: cephadm 2024-03-07T03:08:55.974034+0000 mgr.y (mgr.24287) 14 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:08:58.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: cephadm 2024-03-07T03:08:55.976679+0000 mgr.y (mgr.24287) 15 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:08:58.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: cephadm 2024-03-07T03:08:56.139441+0000 mgr.y (mgr.24287) 16 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:08:58.060 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: cephadm 2024-03-07T03:08:56.141697+0000 mgr.y (mgr.24287) 17 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:08:58.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: cephadm 2024-03-07T03:08:56.304208+0000 mgr.y (mgr.24287) 18 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:08:58.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: cephadm 2024-03-07T03:08:56.306383+0000 mgr.y (mgr.24287) 19 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:08:58.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: cephadm 2024-03-07T03:08:56.527873+0000 mgr.y (mgr.24287) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi148 2024-03-07T03:08:58.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: audit 2024-03-07T03:08:57.484889+0000 mon.a (mon.0) 686 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: audit 2024-03-07T03:08:57.493316+0000 mon.a (mon.0) 687 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:57 smithi195 bash[23221]: audit 2024-03-07T03:08:57.550819+0000 mon.a (mon.0) 688 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.318 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.319 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.319 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.319 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.319 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.320 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.540 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: v1.5.0: Pulling from prometheus/node-exporter 2024-03-07T03:08:58.541 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: 22b70bddd3ac: Pulling fs layer 2024-03-07T03:08:58.541 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: 5c12815fee55: Pulling fs layer 2024-03-07T03:08:58.541 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: c0e87333d380: Pulling fs layer 2024-03-07T03:08:58.584 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.584 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.585 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.585 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.585 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.585 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.586 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.586 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.586 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.586 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:58.586 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:58 smithi195 systemd[1]: Started Ceph node-exporter.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:08:58.730 INFO:teuthology.orchestra.run.smithi195.stdout:Scheduled grafana update... 2024-03-07T03:08:58.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[21362]: cluster 2024-03-07T03:08:56.967998+0000 mgr.y (mgr.24287) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-07T03:08:58.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[21362]: cephadm 2024-03-07T03:08:57.554202+0000 mgr.y (mgr.24287) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi195 2024-03-07T03:08:58.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[21362]: audit 2024-03-07T03:08:58.641914+0000 mon.a (mon.0) 689 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[21362]: audit 2024-03-07T03:08:58.657519+0000 mon.a (mon.0) 690 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[31149]: cluster 2024-03-07T03:08:56.967998+0000 mgr.y (mgr.24287) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-07T03:08:58.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[31149]: cephadm 2024-03-07T03:08:57.554202+0000 mgr.y (mgr.24287) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi195 2024-03-07T03:08:58.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[31149]: audit 2024-03-07T03:08:58.641914+0000 mon.a (mon.0) 689 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[31149]: audit 2024-03-07T03:08:58.657519+0000 mon.a (mon.0) 690 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.795 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: 5c12815fee55: Verifying Checksum 2024-03-07T03:08:58.795 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: 5c12815fee55: Download complete 2024-03-07T03:08:58.795 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: 22b70bddd3ac: Verifying Checksum 2024-03-07T03:08:58.795 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: 22b70bddd3ac: Download complete 2024-03-07T03:08:58.970 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:58 smithi195 bash[48209]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-03-07T03:08:58.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:58 smithi195 bash[23221]: cluster 2024-03-07T03:08:56.967998+0000 mgr.y (mgr.24287) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-07T03:08:58.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:58 smithi195 bash[23221]: cephadm 2024-03-07T03:08:57.554202+0000 mgr.y (mgr.24287) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi195 2024-03-07T03:08:58.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:58 smithi195 bash[23221]: audit 2024-03-07T03:08:58.641914+0000 mon.a (mon.0) 689 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:58.970 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:58 smithi195 bash[23221]: audit 2024-03-07T03:08:58.657519+0000 mon.a (mon.0) 690 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.192 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: 22b70bddd3ac: Pull complete 2024-03-07T03:08:59.192 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: c0e87333d380: Verifying Checksum 2024-03-07T03:08:59.192 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:58 smithi148 bash[55889]: c0e87333d380: Download complete 2024-03-07T03:08:59.192 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:08:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:08:59.934 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: audit 2024-03-07T03:08:58.681697+0000 mon.a (mon.0) 691 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.935 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: audit 2024-03-07T03:08:58.699760+0000 mon.a (mon.0) 692 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.935 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: audit 2024-03-07T03:08:58.711943+0000 mon.a (mon.0) 693 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.935 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: cephadm 2024-03-07T03:08:58.727416+0000 mgr.y (mgr.24287) 23 : cephadm 1 Deploying daemon alertmanager.a on smithi148 2024-03-07T03:08:59.935 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: audit 2024-03-07T03:08:58.728837+0000 mgr.y (mgr.24287) 24 : audit 0 from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:59.935 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: cephadm 2024-03-07T03:08:58.730985+0000 mgr.y (mgr.24287) 25 : cephadm 1 Saving service grafana spec with placement smithi195=a;count:1 2024-03-07T03:08:59.935 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: audit 2024-03-07T03:08:58.738056+0000 mon.a (mon.0) 694 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.935 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[23221]: audit 2024-03-07T03:08:59.038996+0000 mon.a (mon.0) 695 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.935 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:08:59 smithi195 bash[48209]: v1.5.0: Pulling from prometheus/node-exporter 2024-03-07T03:08:59.936 DEBUG:teuthology.orchestra.run.smithi195:grafana.a> sudo journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@grafana.a.service 2024-03-07T03:08:59.939 INFO:tasks.cephadm:Setting up client nodes... 2024-03-07T03:08:59.939 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-07T03:08:59.945 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[55889]: 5c12815fee55: Pull complete 2024-03-07T03:08:59.945 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[55889]: c0e87333d380: Pull complete 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: audit 2024-03-07T03:08:58.681697+0000 mon.a (mon.0) 691 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: audit 2024-03-07T03:08:58.699760+0000 mon.a (mon.0) 692 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: audit 2024-03-07T03:08:58.711943+0000 mon.a (mon.0) 693 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: cephadm 2024-03-07T03:08:58.727416+0000 mgr.y (mgr.24287) 23 : cephadm 1 Deploying daemon alertmanager.a on smithi148 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: audit 2024-03-07T03:08:58.728837+0000 mgr.y (mgr.24287) 24 : audit 0 from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: cephadm 2024-03-07T03:08:58.730985+0000 mgr.y (mgr.24287) 25 : cephadm 1 Saving service grafana spec with placement smithi195=a;count:1 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: audit 2024-03-07T03:08:58.738056+0000 mon.a (mon.0) 694 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.946 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[21362]: audit 2024-03-07T03:08:59.038996+0000 mon.a (mon.0) 695 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: audit 2024-03-07T03:08:58.681697+0000 mon.a (mon.0) 691 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: audit 2024-03-07T03:08:58.699760+0000 mon.a (mon.0) 692 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: audit 2024-03-07T03:08:58.711943+0000 mon.a (mon.0) 693 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: cephadm 2024-03-07T03:08:58.727416+0000 mgr.y (mgr.24287) 23 : cephadm 1 Deploying daemon alertmanager.a on smithi148 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: audit 2024-03-07T03:08:58.728837+0000 mgr.y (mgr.24287) 24 : audit 0 from='client.24331 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi195=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: cephadm 2024-03-07T03:08:58.730985+0000 mgr.y (mgr.24287) 25 : cephadm 1 Saving service grafana spec with placement smithi195=a;count:1 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: audit 2024-03-07T03:08:58.738056+0000 mon.a (mon.0) 694 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:08:59.947 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[31149]: audit 2024-03-07T03:08:59.038996+0000 mon.a (mon.0) 695 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:00.226 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 22b70bddd3ac: Pulling fs layer 2024-03-07T03:09:00.226 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 5c12815fee55: Pulling fs layer 2024-03-07T03:09:00.226 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: c0e87333d380: Pulling fs layer 2024-03-07T03:09:00.295 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[55889]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-03-07T03:09:00.295 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:08:59 smithi148 bash[55889]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-03-07T03:09:00.318 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 5c12815fee55: Verifying Checksum 2024-03-07T03:09:00.318 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 5c12815fee55: Download complete 2024-03-07T03:09:00.318 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 22b70bddd3ac: Verifying Checksum 2024-03-07T03:09:00.318 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 22b70bddd3ac: Download complete 2024-03-07T03:09:00.730 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 22b70bddd3ac: Pull complete 2024-03-07T03:09:00.730 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: c0e87333d380: Verifying Checksum 2024-03-07T03:09:00.730 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: c0e87333d380: Download complete 2024-03-07T03:09:01.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:00 smithi148 bash[21362]: cluster 2024-03-07T03:08:58.968416+0000 mgr.y (mgr.24287) 26 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-07T03:09:01.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:00 smithi148 bash[31149]: cluster 2024-03-07T03:08:58.968416+0000 mgr.y (mgr.24287) 26 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-07T03:09:01.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[23221]: cluster 2024-03-07T03:08:58.968416+0000 mgr.y (mgr.24287) 26 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-07T03:09:01.068 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:00 smithi195 bash[48209]: 5c12815fee55: Pull complete 2024-03-07T03:09:01.464 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:01 smithi195 bash[48209]: c0e87333d380: Pull complete 2024-03-07T03:09:01.464 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:01 smithi195 bash[48209]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-03-07T03:09:01.465 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:01 smithi195 bash[48209]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-03-07T03:09:02.045 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.806Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-07T03:09:02.045 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.806Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-07T03:09:02.045 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.807Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-07T03:09:02.045 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.807Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-07T03:09:02.045 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.807Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-07T03:09:02.045 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=arp 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=edac 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-07T03:09:02.046 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-07T03:09:02.047 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=os 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-07T03:09:02.048 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=stat 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=time 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=uname 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-07T03:09:02.049 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:01 smithi148 bash[55889]: ts=2024-03-07T03:09:01.808Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-07T03:09:03.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:02 smithi148 bash[21362]: cluster 2024-03-07T03:09:00.969079+0000 mgr.y (mgr.24287) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-07T03:09:03.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:02 smithi148 bash[31149]: cluster 2024-03-07T03:09:00.969079+0000 mgr.y (mgr.24287) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-07T03:09:03.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[23221]: cluster 2024-03-07T03:09:00.969079+0000 mgr.y (mgr.24287) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-07T03:09:03.069 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.827Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-07T03:09:03.069 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.827Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-07T03:09:03.069 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.828Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-07T03:09:03.069 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.828Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-07T03:09:03.069 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.828Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-07T03:09:03.069 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.828Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-07T03:09:03.069 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=arp 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=edac 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-07T03:09:03.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-07T03:09:03.071 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=os 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=stat 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-07T03:09:03.072 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=time 2024-03-07T03:09:03.073 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-07T03:09:03.073 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=uname 2024-03-07T03:09:03.073 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-07T03:09:03.073 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-07T03:09:03.073 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-07T03:09:03.073 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.829Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-07T03:09:03.073 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:02 smithi195 bash[48209]: ts=2024-03-07T03:09:02.830Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-07T03:09:05.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:04 smithi148 bash[21362]: cluster 2024-03-07T03:09:02.969637+0000 mgr.y (mgr.24287) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:05.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:04 smithi148 bash[21362]: audit 2024-03-07T03:09:04.043376+0000 mon.b (mon.1) 57 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:05.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:04 smithi148 bash[31149]: cluster 2024-03-07T03:09:02.969637+0000 mgr.y (mgr.24287) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:05.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:04 smithi148 bash[31149]: audit 2024-03-07T03:09:04.043376+0000 mon.b (mon.1) 57 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:05.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:04 smithi195 bash[23221]: cluster 2024-03-07T03:09:02.969637+0000 mgr.y (mgr.24287) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:05.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:04 smithi195 bash[23221]: audit 2024-03-07T03:09:04.043376+0000 mon.b (mon.1) 57 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:06.017 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:06.912 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:06 smithi148 bash[21362]: cluster 2024-03-07T03:09:04.970081+0000 mgr.y (mgr.24287) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:06.913 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:06 smithi148 bash[31149]: cluster 2024-03-07T03:09:04.970081+0000 mgr.y (mgr.24287) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:07.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:06 smithi195 bash[23221]: cluster 2024-03-07T03:09:04.970081+0000 mgr.y (mgr.24287) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:08.418 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.419 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.419 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.419 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.420 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.420 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.420 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.420 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.421 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.421 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.670 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.671 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.671 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.671 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.671 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.672 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.672 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.672 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.673 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:08.673 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:09.044 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:08 smithi148 systemd[1]: Started Ceph alertmanager.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:09:09.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:08 smithi148 bash[31149]: cluster 2024-03-07T03:09:06.970501+0000 mgr.y (mgr.24287) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:09.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:08 smithi148 bash[31149]: audit 2024-03-07T03:09:08.718270+0000 mon.a (mon.0) 696 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:09.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:08 smithi148 bash[31149]: audit 2024-03-07T03:09:08.731289+0000 mon.a (mon.0) 697 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:09.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:08 smithi148 bash[21362]: cluster 2024-03-07T03:09:06.970501+0000 mgr.y (mgr.24287) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:09.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:08 smithi148 bash[21362]: audit 2024-03-07T03:09:08.718270+0000 mon.a (mon.0) 696 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:09.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:08 smithi148 bash[21362]: audit 2024-03-07T03:09:08.731289+0000 mon.a (mon.0) 697 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:09.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:08 smithi195 bash[23221]: cluster 2024-03-07T03:09:06.970501+0000 mgr.y (mgr.24287) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-07T03:09:09.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:08 smithi195 bash[23221]: audit 2024-03-07T03:09:08.718270+0000 mon.a (mon.0) 696 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:09.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:08 smithi195 bash[23221]: audit 2024-03-07T03:09:08.731289+0000 mon.a (mon.0) 697 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[21362]: audit 2024-03-07T03:09:08.771864+0000 mon.a (mon.0) 698 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[21362]: audit 2024-03-07T03:09:08.780856+0000 mon.a (mon.0) 699 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[21362]: audit 2024-03-07T03:09:08.830871+0000 mon.b (mon.1) 58 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[21362]: audit 2024-03-07T03:09:09.065360+0000 mon.a (mon.0) 700 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[31149]: audit 2024-03-07T03:09:08.771864+0000 mon.a (mon.0) 698 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[31149]: audit 2024-03-07T03:09:08.780856+0000 mon.a (mon.0) 699 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[31149]: audit 2024-03-07T03:09:08.830871+0000 mon.b (mon.1) 58 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:10.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:09 smithi148 bash[31149]: audit 2024-03-07T03:09:09.065360+0000 mon.a (mon.0) 700 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:09 smithi195 bash[23221]: audit 2024-03-07T03:09:08.771864+0000 mon.a (mon.0) 698 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:09 smithi195 bash[23221]: audit 2024-03-07T03:09:08.780856+0000 mon.a (mon.0) 699 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:09 smithi195 bash[23221]: audit 2024-03-07T03:09:08.830871+0000 mon.b (mon.1) 58 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:10.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:09 smithi195 bash[23221]: audit 2024-03-07T03:09:09.065360+0000 mon.a (mon.0) 700 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:10.414 INFO:teuthology.orchestra.run.smithi148.stdout:[client.0] 2024-03-07T03:09:10.414 INFO:teuthology.orchestra.run.smithi148.stdout: key = AQDWL+ll/lCYGBAAGR4Tfjyv+g/9UXg4uAjeVw== 2024-03-07T03:09:10.439 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.278Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-07T03:09:10.439 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.279Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-07T03:09:10.439 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.280Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.148 port=9094 2024-03-07T03:09:10.440 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.281Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-07T03:09:10.440 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.310Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:09:10.440 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.311Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:09:10.440 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.314Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-07T03:09:10.440 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[56354]: ts=2024-03-07T03:09:10.314Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-07T03:09:11.006 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T03:09:11.006 DEBUG:teuthology.orchestra.run.smithi148:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-07T03:09:11.006 DEBUG:teuthology.orchestra.run.smithi148:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-07T03:09:11.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[21362]: cluster 2024-03-07T03:09:08.970894+0000 mgr.y (mgr.24287) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:11.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[21362]: audit 2024-03-07T03:09:10.411581+0000 mon.c (mon.2) 19 : audit 1 from='client.? 172.21.15.148:0/3738254251' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:11.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[21362]: audit 2024-03-07T03:09:10.412314+0000 mon.a (mon.0) 701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:11.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[21362]: audit 2024-03-07T03:09:10.420502+0000 mon.a (mon.0) 702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:09:11.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[31149]: cluster 2024-03-07T03:09:08.970894+0000 mgr.y (mgr.24287) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:11.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[31149]: audit 2024-03-07T03:09:10.411581+0000 mon.c (mon.2) 19 : audit 1 from='client.? 172.21.15.148:0/3738254251' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:11.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[31149]: audit 2024-03-07T03:09:10.412314+0000 mon.a (mon.0) 701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:11.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:10 smithi148 bash[31149]: audit 2024-03-07T03:09:10.420502+0000 mon.a (mon.0) 702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:09:11.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:10 smithi195 bash[23221]: cluster 2024-03-07T03:09:08.970894+0000 mgr.y (mgr.24287) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:11.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:10 smithi195 bash[23221]: audit 2024-03-07T03:09:10.411581+0000 mon.c (mon.2) 19 : audit 1 from='client.? 172.21.15.148:0/3738254251' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:11.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:10 smithi195 bash[23221]: audit 2024-03-07T03:09:10.412314+0000 mon.a (mon.0) 701 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:11.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:10 smithi195 bash[23221]: audit 2024-03-07T03:09:10.420502+0000 mon.a (mon.0) 702 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:09:11.069 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-07T03:09:12.544 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:12 smithi148 bash[56354]: ts=2024-03-07T03:09:12.281Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000021048s 2024-03-07T03:09:13.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:12 smithi148 bash[21362]: cluster 2024-03-07T03:09:10.971487+0000 mgr.y (mgr.24287) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:13.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:12 smithi148 bash[31149]: cluster 2024-03-07T03:09:10.971487+0000 mgr.y (mgr.24287) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:13.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:12 smithi195 bash[23221]: cluster 2024-03-07T03:09:10.971487+0000 mgr.y (mgr.24287) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:15.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:14 smithi148 bash[21362]: cluster 2024-03-07T03:09:12.972089+0000 mgr.y (mgr.24287) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:15.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:14 smithi148 bash[21362]: audit 2024-03-07T03:09:14.221668+0000 mon.a (mon.0) 703 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:15.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:14 smithi148 bash[21362]: audit 2024-03-07T03:09:14.232016+0000 mon.a (mon.0) 704 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:15.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:14 smithi148 bash[31149]: cluster 2024-03-07T03:09:12.972089+0000 mgr.y (mgr.24287) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:15.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:14 smithi148 bash[31149]: audit 2024-03-07T03:09:14.221668+0000 mon.a (mon.0) 703 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:15.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:14 smithi148 bash[31149]: audit 2024-03-07T03:09:14.232016+0000 mon.a (mon.0) 704 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:15.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:14 smithi195 bash[23221]: cluster 2024-03-07T03:09:12.972089+0000 mgr.y (mgr.24287) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:15.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:14 smithi195 bash[23221]: audit 2024-03-07T03:09:14.221668+0000 mon.a (mon.0) 703 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:15.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:14 smithi195 bash[23221]: audit 2024-03-07T03:09:14.232016+0000 mon.a (mon.0) 704 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:15.707 INFO:teuthology.orchestra.run.smithi195.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.b/config 2024-03-07T03:09:16.044 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:15 smithi148 bash[21615]: [07/Mar/2024:03:09:15] ENGINE Bus STOPPING 2024-03-07T03:09:16.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:15 smithi148 bash[21615]: [07/Mar/2024:03:09:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:09:16.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:15 smithi148 bash[21615]: [07/Mar/2024:03:09:15] ENGINE Bus STOPPED 2024-03-07T03:09:16.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:15 smithi148 bash[21615]: [07/Mar/2024:03:09:15] ENGINE Bus STARTING 2024-03-07T03:09:16.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:15 smithi148 bash[21615]: [07/Mar/2024:03:09:15] ENGINE Serving on http://:::9283 2024-03-07T03:09:16.045 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:15 smithi148 bash[21615]: [07/Mar/2024:03:09:15] ENGINE Bus STARTED 2024-03-07T03:09:16.265 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:16 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: cluster 2024-03-07T03:09:14.972471+0000 mgr.y (mgr.24287) 34 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.272313+0000 mon.a (mon.0) 705 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.283942+0000 mon.a (mon.0) 706 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.287220+0000 mon.b (mon.1) 59 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.288890+0000 mon.b (mon.1) 60 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.297304+0000 mon.a (mon.0) 707 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: cephadm 2024-03-07T03:09:15.644865+0000 mgr.y (mgr.24287) 35 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.705662+0000 mon.a (mon.0) 708 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.713843+0000 mon.a (mon.0) 709 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.716041+0000 mon.b (mon.1) 61 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.718928+0000 mgr.y (mgr.24287) 36 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: audit 2024-03-07T03:09:15.728503+0000 mon.a (mon.0) 710 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[21362]: cephadm 2024-03-07T03:09:15.737570+0000 mgr.y (mgr.24287) 37 : cephadm 1 Deploying daemon grafana.a on smithi195 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: cluster 2024-03-07T03:09:14.972471+0000 mgr.y (mgr.24287) 34 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.272313+0000 mon.a (mon.0) 705 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.283942+0000 mon.a (mon.0) 706 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.287220+0000 mon.b (mon.1) 59 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.288890+0000 mon.b (mon.1) 60 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.297304+0000 mon.a (mon.0) 707 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: cephadm 2024-03-07T03:09:15.644865+0000 mgr.y (mgr.24287) 35 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.705662+0000 mon.a (mon.0) 708 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.713843+0000 mon.a (mon.0) 709 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.716041+0000 mon.b (mon.1) 61 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:09:16.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.718928+0000 mgr.y (mgr.24287) 36 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:09:16.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: audit 2024-03-07T03:09:15.728503+0000 mon.a (mon.0) 710 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:16 smithi148 bash[31149]: cephadm 2024-03-07T03:09:15.737570+0000 mgr.y (mgr.24287) 37 : cephadm 1 Deploying daemon grafana.a on smithi195 2024-03-07T03:09:16.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: cluster 2024-03-07T03:09:14.972471+0000 mgr.y (mgr.24287) 34 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.272313+0000 mon.a (mon.0) 705 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.283942+0000 mon.a (mon.0) 706 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.287220+0000 mon.b (mon.1) 59 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.288890+0000 mon.b (mon.1) 60 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.297304+0000 mon.a (mon.0) 707 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: cephadm 2024-03-07T03:09:15.644865+0000 mgr.y (mgr.24287) 35 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.705662+0000 mon.a (mon.0) 708 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.713843+0000 mon.a (mon.0) 709 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.716041+0000 mon.b (mon.1) 61 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.718928+0000 mgr.y (mgr.24287) 36 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: audit 2024-03-07T03:09:15.728503+0000 mon.a (mon.0) 710 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:16 smithi195 bash[23221]: cephadm 2024-03-07T03:09:15.737570+0000 mgr.y (mgr.24287) 37 : cephadm 1 Deploying daemon grafana.a on smithi195 2024-03-07T03:09:18.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:18 smithi148 bash[21362]: cluster 2024-03-07T03:09:16.973025+0000 mgr.y (mgr.24287) 38 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:18 smithi148 bash[31149]: cluster 2024-03-07T03:09:16.973025+0000 mgr.y (mgr.24287) 38 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:18.632 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:18 smithi195 bash[23221]: cluster 2024-03-07T03:09:16.973025+0000 mgr.y (mgr.24287) 38 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:19.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:19 smithi148 bash[21362]: audit 2024-03-07T03:09:19.043412+0000 mon.b (mon.1) 62 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:19.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:19 smithi148 bash[31149]: audit 2024-03-07T03:09:19.043412+0000 mon.b (mon.1) 62 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:19.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:19 smithi195 bash[23221]: audit 2024-03-07T03:09:19.043412+0000 mon.b (mon.1) 62 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:19.957 INFO:teuthology.orchestra.run.smithi195.stdout:[client.1] 2024-03-07T03:09:19.957 INFO:teuthology.orchestra.run.smithi195.stdout: key = AQDfL+llIksqORAAFnC6q8qwFBASytksJFjGEA== 2024-03-07T03:09:20.544 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[56354]: ts=2024-03-07T03:09:20.284Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002759376s 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[31149]: cluster 2024-03-07T03:09:18.973627+0000 mgr.y (mgr.24287) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[31149]: audit 2024-03-07T03:09:19.956079+0000 mon.b (mon.1) 63 : audit 1 from='client.? 172.21.15.195:0/3818317643' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[31149]: audit 2024-03-07T03:09:19.958689+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[31149]: audit 2024-03-07T03:09:19.966402+0000 mon.a (mon.0) 712 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[21362]: cluster 2024-03-07T03:09:18.973627+0000 mgr.y (mgr.24287) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[21362]: audit 2024-03-07T03:09:19.956079+0000 mon.b (mon.1) 63 : audit 1 from='client.? 172.21.15.195:0/3818317643' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[21362]: audit 2024-03-07T03:09:19.958689+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:20.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:20 smithi148 bash[21362]: audit 2024-03-07T03:09:19.966402+0000 mon.a (mon.0) 712 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:09:20.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:20 smithi195 bash[23221]: cluster 2024-03-07T03:09:18.973627+0000 mgr.y (mgr.24287) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:20.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:20 smithi195 bash[23221]: audit 2024-03-07T03:09:19.956079+0000 mon.b (mon.1) 63 : audit 1 from='client.? 172.21.15.195:0/3818317643' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:20.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:20 smithi195 bash[23221]: audit 2024-03-07T03:09:19.958689+0000 mon.a (mon.0) 711 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-07T03:09:20.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:20 smithi195 bash[23221]: audit 2024-03-07T03:09:19.966402+0000 mon.a (mon.0) 712 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-07T03:09:21.433 DEBUG:teuthology.orchestra.run.smithi195:> set -ex 2024-03-07T03:09:21.433 DEBUG:teuthology.orchestra.run.smithi195:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-03-07T03:09:21.433 DEBUG:teuthology.orchestra.run.smithi195:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-03-07T03:09:21.453 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-03-07T03:09:21.453 DEBUG:teuthology.orchestra.run.smithi148:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-07T03:09:21.758 INFO:teuthology.misc.health.smithi148.stdout: 2024-03-07T03:09:21.758 INFO:teuthology.misc.health.smithi148.stdout:{"epoch":60,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:08:48.946859+0000","last_up_change":"2024-03-07T03:07:53.161981+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6803","nonce":1285694746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6805","nonce":1285694746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6809","nonce":1285694746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6807","nonce":1285694746}]},"public_addr":"172.21.15.148:6803/1285694746","cluster_addr":"172.21.15.148:6805/1285694746","heartbeat_back_addr":"172.21.15.148:6809/1285694746","heartbeat_front_addr":"172.21.15.148:6807/1285694746","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6811","nonce":3763364013}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6813","nonce":3763364013}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6817","nonce":3763364013}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6815","nonce":3763364013}]},"public_addr":"172.21.15.148:6811/3763364013","cluster_addr":"172.21.15.148:6813/3763364013","heartbeat_back_addr":"172.21.15.148:6817/3763364013","heartbeat_front_addr":"172.21.15.148:6815/3763364013","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6819","nonce":916957575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6821","nonce":916957575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6825","nonce":916957575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6823","nonce":916957575}]},"public_addr":"172.21.15.148:6819/916957575","cluster_addr":"172.21.15.148:6821/916957575","heartbeat_back_addr":"172.21.15.148:6825/916957575","heartbeat_front_addr":"172.21.15.148:6823/916957575","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6827","nonce":2267330328}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6829","nonce":2267330328}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6833","nonce":2267330328}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6831","nonce":2267330328}]},"public_addr":"172.21.15.148:6827/2267330328","cluster_addr":"172.21.15.148:6829/2267330328","heartbeat_back_addr":"172.21.15.148:6833/2267330328","heartbeat_front_addr":"172.21.15.148:6831/2267330328","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6801","nonce":1363408518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6803","nonce":1363408518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6807","nonce":1363408518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6805","nonce":1363408518}]},"public_addr":"172.21.15.195:6801/1363408518","cluster_addr":"172.21.15.195:6803/1363408518","heartbeat_back_addr":"172.21.15.195:6807/1363408518","heartbeat_front_addr":"172.21.15.195:6805/1363408518","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:6801/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/1901736382":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/4143961239":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/3220133019":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2258249874":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:09:21.776 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-03-07T03:09:21.776 INFO:tasks.cephadm:Creating RBD pool 2024-03-07T03:09:21.776 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-03-07T03:09:22.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:22 smithi148 bash[21362]: cluster 2024-03-07T03:09:20.974250+0000 mgr.y (mgr.24287) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:22.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:22 smithi148 bash[21362]: audit 2024-03-07T03:09:21.765712+0000 mon.a (mon.0) 713 : audit 0 from='client.? 172.21.15.148:0/2437725300' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:22.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:22 smithi148 bash[31149]: cluster 2024-03-07T03:09:20.974250+0000 mgr.y (mgr.24287) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:22.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:22 smithi148 bash[31149]: audit 2024-03-07T03:09:21.765712+0000 mon.a (mon.0) 713 : audit 0 from='client.? 172.21.15.148:0/2437725300' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:22.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:22 smithi195 bash[23221]: cluster 2024-03-07T03:09:20.974250+0000 mgr.y (mgr.24287) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:22.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:22 smithi195 bash[23221]: audit 2024-03-07T03:09:21.765712+0000 mon.a (mon.0) 713 : audit 0 from='client.? 172.21.15.148:0/2437725300' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:24.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:24 smithi148 bash[21362]: cluster 2024-03-07T03:09:22.974807+0000 mgr.y (mgr.24287) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:24.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:24 smithi148 bash[31149]: cluster 2024-03-07T03:09:22.974807+0000 mgr.y (mgr.24287) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:24.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:24 smithi195 bash[23221]: cluster 2024-03-07T03:09:22.974807+0000 mgr.y (mgr.24287) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:26.501 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:26.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:26 smithi148 bash[21362]: cluster 2024-03-07T03:09:24.975320+0000 mgr.y (mgr.24287) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:26.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:26 smithi148 bash[31149]: cluster 2024-03-07T03:09:24.975320+0000 mgr.y (mgr.24287) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:26.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:26 smithi195 bash[23221]: cluster 2024-03-07T03:09:24.975320+0000 mgr.y (mgr.24287) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:28.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:28 smithi148 bash[21362]: cluster 2024-03-07T03:09:26.975939+0000 mgr.y (mgr.24287) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:28.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:28 smithi148 bash[31149]: cluster 2024-03-07T03:09:26.975939+0000 mgr.y (mgr.24287) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:28.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:28 smithi195 bash[23221]: cluster 2024-03-07T03:09:26.975939+0000 mgr.y (mgr.24287) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:30.317 INFO:teuthology.orchestra.run.smithi148.stderr:pool 'rbd' created 2024-03-07T03:09:30.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:30 smithi195 bash[23221]: cluster 2024-03-07T03:09:28.976559+0000 mgr.y (mgr.24287) 44 : cluster 0 pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:30.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:30 smithi195 bash[23221]: audit 2024-03-07T03:09:29.413472+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.148:0/518905204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:09:30.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:30 smithi195 bash[23221]: audit 2024-03-07T03:09:29.413915+0000 mon.a (mon.0) 714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:09:30.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:30 smithi148 bash[21362]: cluster 2024-03-07T03:09:28.976559+0000 mgr.y (mgr.24287) 44 : cluster 0 pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:30.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:30 smithi148 bash[21362]: audit 2024-03-07T03:09:29.413472+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.148:0/518905204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:09:30.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:30 smithi148 bash[21362]: audit 2024-03-07T03:09:29.413915+0000 mon.a (mon.0) 714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:09:30.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:30 smithi148 bash[31149]: cluster 2024-03-07T03:09:28.976559+0000 mgr.y (mgr.24287) 44 : cluster 0 pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:30.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:30 smithi148 bash[31149]: audit 2024-03-07T03:09:29.413472+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.148:0/518905204' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:09:30.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:30 smithi148 bash[31149]: audit 2024-03-07T03:09:29.413915+0000 mon.a (mon.0) 714 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-07T03:09:30.960 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-03-07T03:09:31.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:31 smithi195 bash[23221]: audit 2024-03-07T03:09:30.320903+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-07T03:09:31.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:31 smithi195 bash[23221]: cluster 2024-03-07T03:09:30.321076+0000 mon.a (mon.0) 716 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-03-07T03:09:31.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:31 smithi148 bash[21362]: audit 2024-03-07T03:09:30.320903+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-07T03:09:31.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:31 smithi148 bash[21362]: cluster 2024-03-07T03:09:30.321076+0000 mon.a (mon.0) 716 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-03-07T03:09:31.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:31 smithi148 bash[31149]: audit 2024-03-07T03:09:30.320903+0000 mon.a (mon.0) 715 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-07T03:09:31.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:31 smithi148 bash[31149]: cluster 2024-03-07T03:09:30.321076+0000 mon.a (mon.0) 716 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-03-07T03:09:32.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:32 smithi148 bash[21362]: cluster 2024-03-07T03:09:30.977154+0000 mgr.y (mgr.24287) 45 : cluster 0 pgmap v25: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:32.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:32 smithi148 bash[21362]: cluster 2024-03-07T03:09:31.315486+0000 mon.a (mon.0) 717 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:09:32.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:32 smithi148 bash[21362]: cluster 2024-03-07T03:09:31.335645+0000 mon.a (mon.0) 718 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-03-07T03:09:32.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:32 smithi148 bash[31149]: cluster 2024-03-07T03:09:30.977154+0000 mgr.y (mgr.24287) 45 : cluster 0 pgmap v25: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:32.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:32 smithi148 bash[31149]: cluster 2024-03-07T03:09:31.315486+0000 mon.a (mon.0) 717 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:09:32.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:32 smithi148 bash[31149]: cluster 2024-03-07T03:09:31.335645+0000 mon.a (mon.0) 718 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-03-07T03:09:32.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:32 smithi195 bash[23221]: cluster 2024-03-07T03:09:30.977154+0000 mgr.y (mgr.24287) 45 : cluster 0 pgmap v25: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:32.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:32 smithi195 bash[23221]: cluster 2024-03-07T03:09:31.315486+0000 mon.a (mon.0) 717 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:09:32.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:32 smithi195 bash[23221]: cluster 2024-03-07T03:09:31.335645+0000 mon.a (mon.0) 718 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-03-07T03:09:33.768 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:33 smithi195 bash[23221]: cluster 2024-03-07T03:09:32.344537+0000 mon.a (mon.0) 719 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-03-07T03:09:33.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:33 smithi148 bash[21362]: cluster 2024-03-07T03:09:32.344537+0000 mon.a (mon.0) 719 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-03-07T03:09:33.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:33 smithi148 bash[31149]: cluster 2024-03-07T03:09:32.344537+0000 mon.a (mon.0) 719 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-03-07T03:09:34.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:34 smithi148 bash[21362]: cluster 2024-03-07T03:09:32.977872+0000 mgr.y (mgr.24287) 46 : cluster 0 pgmap v28: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:34.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:34 smithi148 bash[21362]: audit 2024-03-07T03:09:34.053172+0000 mon.a (mon.0) 720 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:34.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:34 smithi148 bash[21362]: audit 2024-03-07T03:09:34.053786+0000 mon.b (mon.1) 64 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:34.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:34 smithi148 bash[31149]: cluster 2024-03-07T03:09:32.977872+0000 mgr.y (mgr.24287) 46 : cluster 0 pgmap v28: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:34.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:34 smithi148 bash[31149]: audit 2024-03-07T03:09:34.053172+0000 mon.a (mon.0) 720 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:34.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:34 smithi148 bash[31149]: audit 2024-03-07T03:09:34.053786+0000 mon.b (mon.1) 64 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:34.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:34 smithi195 bash[23221]: cluster 2024-03-07T03:09:32.977872+0000 mgr.y (mgr.24287) 46 : cluster 0 pgmap v28: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:34.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:34 smithi195 bash[23221]: audit 2024-03-07T03:09:34.053172+0000 mon.a (mon.0) 720 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:34.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:34 smithi195 bash[23221]: audit 2024-03-07T03:09:34.053786+0000 mon.b (mon.1) 64 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:35.689 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:36.623 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:36 smithi148 bash[21362]: cluster 2024-03-07T03:09:34.978531+0000 mgr.y (mgr.24287) 47 : cluster 0 pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:36.623 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:36 smithi148 bash[31149]: cluster 2024-03-07T03:09:34.978531+0000 mgr.y (mgr.24287) 47 : cluster 0 pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:36.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:36 smithi195 bash[23221]: cluster 2024-03-07T03:09:34.978531+0000 mgr.y (mgr.24287) 47 : cluster 0 pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:38.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:38 smithi148 bash[21362]: cluster 2024-03-07T03:09:36.979249+0000 mgr.y (mgr.24287) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:38.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:38 smithi148 bash[31149]: cluster 2024-03-07T03:09:36.979249+0000 mgr.y (mgr.24287) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:38 smithi195 bash[23221]: cluster 2024-03-07T03:09:36.979249+0000 mgr.y (mgr.24287) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:39.368 INFO:teuthology.orchestra.run.smithi148.stderr:enabled application 'rbd' on pool 'rbd' 2024-03-07T03:09:39.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:39 smithi148 bash[21362]: audit 2024-03-07T03:09:38.464731+0000 mon.a (mon.0) 721 : audit 1 from='client.? 172.21.15.148:0/3295545032' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:09:39.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:39 smithi148 bash[31149]: audit 2024-03-07T03:09:38.464731+0000 mon.a (mon.0) 721 : audit 1 from='client.? 172.21.15.148:0/3295545032' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:09:39.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:39 smithi195 bash[23221]: audit 2024-03-07T03:09:38.464731+0000 mon.a (mon.0) 721 : audit 1 from='client.? 172.21.15.148:0/3295545032' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:09:40.020 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-07T03:09:40.021 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-07T03:09:40.021 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph mgr dump --format=json 2024-03-07T03:09:40.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:40 smithi148 bash[21362]: cluster 2024-03-07T03:09:38.980027+0000 mgr.y (mgr.24287) 49 : cluster 0 pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:40.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:40 smithi148 bash[21362]: audit 2024-03-07T03:09:39.373551+0000 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.148:0/3295545032' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:09:40.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:40 smithi148 bash[21362]: cluster 2024-03-07T03:09:39.373693+0000 mon.a (mon.0) 723 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-03-07T03:09:40.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:40 smithi148 bash[31149]: cluster 2024-03-07T03:09:38.980027+0000 mgr.y (mgr.24287) 49 : cluster 0 pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:40.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:40 smithi148 bash[31149]: audit 2024-03-07T03:09:39.373551+0000 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.148:0/3295545032' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:09:40.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:40 smithi148 bash[31149]: cluster 2024-03-07T03:09:39.373693+0000 mon.a (mon.0) 723 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-03-07T03:09:40.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:40 smithi195 bash[23221]: cluster 2024-03-07T03:09:38.980027+0000 mgr.y (mgr.24287) 49 : cluster 0 pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:40.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:40 smithi195 bash[23221]: audit 2024-03-07T03:09:39.373551+0000 mon.a (mon.0) 722 : audit 1 from='client.? 172.21.15.148:0/3295545032' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-07T03:09:40.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:40 smithi195 bash[23221]: cluster 2024-03-07T03:09:39.373693+0000 mon.a (mon.0) 723 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-03-07T03:09:41.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:41 smithi148 bash[21362]: cluster 2024-03-07T03:09:41.370396+0000 mon.a (mon.0) 724 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:09:41.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:41 smithi148 bash[21362]: cluster 2024-03-07T03:09:41.370474+0000 mon.a (mon.0) 725 : cluster 1 Cluster is now healthy 2024-03-07T03:09:41.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:41 smithi148 bash[31149]: cluster 2024-03-07T03:09:41.370396+0000 mon.a (mon.0) 724 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:09:41.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:41 smithi148 bash[31149]: cluster 2024-03-07T03:09:41.370474+0000 mon.a (mon.0) 725 : cluster 1 Cluster is now healthy 2024-03-07T03:09:41.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:41 smithi195 bash[23221]: cluster 2024-03-07T03:09:41.370396+0000 mon.a (mon.0) 724 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:09:41.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:41 smithi195 bash[23221]: cluster 2024-03-07T03:09:41.370474+0000 mon.a (mon.0) 725 : cluster 1 Cluster is now healthy 2024-03-07T03:09:42.792 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:42 smithi195 bash[23221]: cluster 2024-03-07T03:09:40.980839+0000 mgr.y (mgr.24287) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:42.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:42 smithi148 bash[21362]: cluster 2024-03-07T03:09:40.980839+0000 mgr.y (mgr.24287) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:42.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:42 smithi148 bash[31149]: cluster 2024-03-07T03:09:40.980839+0000 mgr.y (mgr.24287) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:44.068 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:09:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.068 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:09:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.069 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:09:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.070 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:09:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.070 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:09:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.070 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:43 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.071 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.391 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.392 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.392 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.393 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.393 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.393 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.394 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.394 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.394 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.394 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.394 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.395 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:09:44.395 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:44 smithi195 systemd[1]: Started Ceph grafana.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:09:44.750 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:44.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[21362]: cluster 2024-03-07T03:09:42.981582+0000 mgr.y (mgr.24287) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:44.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[21362]: audit 2024-03-07T03:09:44.097302+0000 mon.a (mon.0) 726 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[21362]: audit 2024-03-07T03:09:44.402550+0000 mon.a (mon.0) 727 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[21362]: audit 2024-03-07T03:09:44.416444+0000 mon.a (mon.0) 728 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[31149]: cluster 2024-03-07T03:09:42.981582+0000 mgr.y (mgr.24287) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:44.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[31149]: audit 2024-03-07T03:09:44.097302+0000 mon.a (mon.0) 726 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[31149]: audit 2024-03-07T03:09:44.402550+0000 mon.a (mon.0) 727 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.771 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:44 smithi148 bash[31149]: audit 2024-03-07T03:09:44.416444+0000 mon.a (mon.0) 728 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:44 smithi195 bash[23221]: cluster 2024-03-07T03:09:42.981582+0000 mgr.y (mgr.24287) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:44.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:44 smithi195 bash[23221]: audit 2024-03-07T03:09:44.097302+0000 mon.a (mon.0) 726 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:44 smithi195 bash[23221]: audit 2024-03-07T03:09:44.402550+0000 mon.a (mon.0) 727 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:44.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:44 smithi195 bash[23221]: audit 2024-03-07T03:09:44.416444+0000 mon.a (mon.0) 728 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.686 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[23221]: audit 2024-03-07T03:09:44.442206+0000 mon.a (mon.0) 729 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.686 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[23221]: audit 2024-03-07T03:09:44.450840+0000 mon.a (mon.0) 730 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.686 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[23221]: audit 2024-03-07T03:09:44.515622+0000 mon.b (mon.1) 65 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:45.686 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[23221]: audit 2024-03-07T03:09:44.893332+0000 mon.a (mon.0) 731 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.686 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[23221]: audit 2024-03-07T03:09:44.900550+0000 mon.a (mon.0) 732 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[21362]: audit 2024-03-07T03:09:44.442206+0000 mon.a (mon.0) 729 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[21362]: audit 2024-03-07T03:09:44.450840+0000 mon.a (mon.0) 730 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[21362]: audit 2024-03-07T03:09:44.515622+0000 mon.b (mon.1) 65 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[21362]: audit 2024-03-07T03:09:44.893332+0000 mon.a (mon.0) 731 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[21362]: audit 2024-03-07T03:09:44.900550+0000 mon.a (mon.0) 732 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[31149]: audit 2024-03-07T03:09:44.442206+0000 mon.a (mon.0) 729 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[31149]: audit 2024-03-07T03:09:44.450840+0000 mon.a (mon.0) 730 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.711 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[31149]: audit 2024-03-07T03:09:44.515622+0000 mon.b (mon.1) 65 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:09:45.712 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[31149]: audit 2024-03-07T03:09:44.893332+0000 mon.a (mon.0) 731 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.712 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:45 smithi148 bash[31149]: audit 2024-03-07T03:09:44.900550+0000 mon.a (mon.0) 732 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:45.940 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693040407Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-03-07T03:09:45.940 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693229078Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693249228Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693257101Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693263799Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693270731Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693277139Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693283768Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693290476Z level=info msg="Path Home" path=/usr/share/grafana 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693297572Z level=info msg="Path Data" path=/var/lib/grafana 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693304547Z level=info msg="Path Logs" path=/var/log/grafana 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693312381Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693318945Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-03-07T03:09:45.941 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=settings t=2024-03-07T03:09:45.693325545Z level=info msg="App mode production" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=sqlstore t=2024-03-07T03:09:45.693387434Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=sqlstore t=2024-03-07T03:09:45.693415956Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.694405827Z level=info msg="Starting DB migrations" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.694497691Z level=info msg="Executing migration" id="create migration_log table" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.727372787Z level=info msg="Executing migration" id="create user table" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.752685265Z level=info msg="Executing migration" id="add unique index user.login" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.769629179Z level=info msg="Executing migration" id="add unique index user.email" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.7948505Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.819952025Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.845612441Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.879333932Z level=info msg="Executing migration" id="create user table v2" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.896233246Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-03-07T03:09:45.942 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.929895701Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-03-07T03:09:45.943 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.946913636Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-03-07T03:09:46.208 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:45 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:45.980423949Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-03-07T03:09:46.211 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.005749456Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-03-07T03:09:46.211 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.039283583Z level=info msg="Executing migration" id="Update user table charset" 2024-03-07T03:09:46.211 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.064588376Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-03-07T03:09:46.211 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.089819097Z level=info msg="Executing migration" id="Add missing user data" 2024-03-07T03:09:46.211 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.11506275Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-03-07T03:09:46.211 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.140291198Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-03-07T03:09:46.212 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.215589556Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-03-07T03:09:46.468 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.257567782Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-03-07T03:09:46.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.2829375Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-03-07T03:09:46.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.308191607Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-03-07T03:09:46.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.350150334Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-03-07T03:09:46.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.392117077Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-03-07T03:09:46.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.434096493Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-03-07T03:09:46.469 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.47605174Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-03-07T03:09:46.746 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.501155167Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-03-07T03:09:46.746 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.543148387Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-03-07T03:09:46.746 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.568522574Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-03-07T03:09:46.746 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.593742095Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-03-07T03:09:46.747 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.619011951Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-03-07T03:09:46.747 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.644350199Z level=info msg="Executing migration" id="create temp_user v2" 2024-03-07T03:09:46.747 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.669495112Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-03-07T03:09:46.747 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.71157515Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-03-07T03:09:46.747 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.753607138Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-03-07T03:09:46.998 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[23221]: cluster 2024-03-07T03:09:44.982177+0000 mgr.y (mgr.24287) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.779083527Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.804450425Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.846328383Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.888134753Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.930387447Z level=info msg="Executing migration" id="create star table" 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.955696256Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:46 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:46.981065113Z level=info msg="Executing migration" id="create org table v1" 2024-03-07T03:09:46.999 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.006294844Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-03-07T03:09:47.251 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.048262609Z level=info msg="Executing migration" id="create org_user table v1" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.073489192Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.098798319Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.124036664Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.149216274Z level=info msg="Executing migration" id="Update org table charset" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.174474972Z level=info msg="Executing migration" id="Update org_user table charset" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.208047079Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.225041207Z level=info msg="Executing migration" id="create dashboard table" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.241978576Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-03-07T03:09:47.252 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.258949077Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-03-07T03:09:47.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:46 smithi148 bash[21362]: cluster 2024-03-07T03:09:44.982177+0000 mgr.y (mgr.24287) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:47.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:46 smithi148 bash[31149]: cluster 2024-03-07T03:09:44.982177+0000 mgr.y (mgr.24287) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:47.503 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.300850424Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-03-07T03:09:47.503 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.359611335Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-03-07T03:09:47.503 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.401508737Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-03-07T03:09:47.503 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.443490207Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-03-07T03:09:47.503 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.468880557Z level=info msg="Executing migration" id="create dashboard v2" 2024-03-07T03:09:47.545 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:09:47.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.510789836Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-03-07T03:09:47.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.552672173Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-03-07T03:09:47.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.594686721Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-03-07T03:09:47.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.620013513Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-03-07T03:09:47.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.645043059Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-03-07T03:09:47.781 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.670752989Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-03-07T03:09:47.782 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.696048359Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-03-07T03:09:47.782 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.721426848Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-03-07T03:09:47.782 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.755050881Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-03-07T03:09:47.782 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.788614612Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-03-07T03:09:48.050 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[23221]: audit 2024-03-07T03:09:47.544625+0000 mon.a (mon.0) 733 : audit 0 from='client.? 172.21.15.148:0/2545345746' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:09:48.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.822204063Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-03-07T03:09:48.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.8475232Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-03-07T03:09:48.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.889442546Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-03-07T03:09:48.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.939909844Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-03-07T03:09:48.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:47 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:47.99011939Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-03-07T03:09:48.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.023925907Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-03-07T03:09:48.050 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.057417078Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-03-07T03:09:48.158 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":20,"active_gid":24287,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6800","nonce":1790474515},{"type":"v1","addr":"172.21.15.148:6801","nonce":1790474515}]},"active_addr":"172.21.15.148:6801/1790474515","active_change":"2024-03-07T03:08:48.947088+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24298,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.148:8443/","prometheus":"http://172.21.15.148:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":2001636836}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":1314748318}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":3482682182}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":2057842025}]}]} 2024-03-07T03:09:48.162 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-07T03:09:48.162 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-07T03:09:48.163 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:09:48.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:47 smithi148 bash[21362]: audit 2024-03-07T03:09:47.544625+0000 mon.a (mon.0) 733 : audit 0 from='client.? 172.21.15.148:0/2545345746' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:09:48.174 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:47 smithi148 bash[31149]: audit 2024-03-07T03:09:47.544625+0000 mon.a (mon.0) 733 : audit 0 from='client.? 172.21.15.148:0/2545345746' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:09:48.318 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.091059042Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-03-07T03:09:48.318 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.124673114Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-03-07T03:09:48.318 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.158290273Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-03-07T03:09:48.318 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.208784749Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-03-07T03:09:48.318 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.233893022Z level=info msg="Executing migration" id="Update dashboard title length" 2024-03-07T03:09:48.318 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.259083313Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-03-07T03:09:48.318 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.301045964Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-03-07T03:09:48.587 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.34311641Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-03-07T03:09:48.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.384996986Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-03-07T03:09:48.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.410341967Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-03-07T03:09:48.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.435511112Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-03-07T03:09:48.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.477550554Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-03-07T03:09:48.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.50280825Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-03-07T03:09:48.588 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.544883745Z level=info msg="Executing migration" id="Add check_sum column" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.595219272Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.628930058Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.671074865Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.704623432Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.746661838Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.7802668Z level=info msg="Executing migration" id="create data_source table" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.813868492Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-03-07T03:09:48.840 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.847448022Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-03-07T03:09:49.109 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[23221]: cluster 2024-03-07T03:09:46.982873+0000 mgr.y (mgr.24287) 53 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:49.109 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.897732776Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-03-07T03:09:49.109 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.948337462Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-03-07T03:09:49.109 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:48 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:48.981935505Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-03-07T03:09:49.109 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.032301708Z level=info msg="Executing migration" id="create data_source table v2" 2024-03-07T03:09:49.109 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.065987071Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-03-07T03:09:49.109 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.082964793Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-03-07T03:09:49.110 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.116637659Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-03-07T03:09:49.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:48 smithi148 bash[21362]: cluster 2024-03-07T03:09:46.982873+0000 mgr.y (mgr.24287) 53 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:49.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:48 smithi148 bash[31149]: cluster 2024-03-07T03:09:46.982873+0000 mgr.y (mgr.24287) 53 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:49.361 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.141751493Z level=info msg="Executing migration" id="Add column with_credentials" 2024-03-07T03:09:49.361 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.167098168Z level=info msg="Executing migration" id="Add secure json data column" 2024-03-07T03:09:49.362 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.209019163Z level=info msg="Executing migration" id="Update data_source table charset" 2024-03-07T03:09:49.362 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.250953167Z level=info msg="Executing migration" id="Update initial version to 1" 2024-03-07T03:09:49.362 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.276367377Z level=info msg="Executing migration" id="Add read_only data column" 2024-03-07T03:09:49.362 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.318296088Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-03-07T03:09:49.362 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.343502391Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-03-07T03:09:49.362 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.368883932Z level=info msg="Executing migration" id="Add uid column" 2024-03-07T03:09:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.410812279Z level=info msg="Executing migration" id="Update uid value" 2024-03-07T03:09:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.452751193Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-03-07T03:09:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.494752648Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-03-07T03:09:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.511784134Z level=info msg="Executing migration" id="create api_key table" 2024-03-07T03:09:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.545492826Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-03-07T03:09:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.587775576Z level=info msg="Executing migration" id="add index api_key.key" 2024-03-07T03:09:49.614 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.604684569Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-03-07T03:09:49.908 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.621606338Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-03-07T03:09:49.908 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.638479638Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-03-07T03:09:49.908 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.655396932Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-03-07T03:09:49.908 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.680657349Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-03-07T03:09:49.908 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.714280603Z level=info msg="Executing migration" id="create api_key table v2" 2024-03-07T03:09:49.908 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.748169606Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-03-07T03:09:49.908 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.773116095Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-03-07T03:09:49.909 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.815131234Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-03-07T03:09:49.909 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.857290689Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-03-07T03:09:49.909 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.915810613Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-03-07T03:09:50.169 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[23221]: cluster 2024-03-07T03:09:48.983423+0000 mgr.y (mgr.24287) 54 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:50.169 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[23221]: audit 2024-03-07T03:09:49.050300+0000 mon.a (mon.0) 734 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.170 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[23221]: audit 2024-03-07T03:09:49.051765+0000 mon.b (mon.1) 66 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:50.170 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[23221]: audit 2024-03-07T03:09:49.104168+0000 mon.a (mon.0) 735 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.170 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[23221]: audit 2024-03-07T03:09:49.431993+0000 mon.a (mon.0) 736 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.170 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[23221]: audit 2024-03-07T03:09:49.441810+0000 mon.a (mon.0) 737 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.170 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:49 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:49.95809805Z level=info msg="Executing migration" id="Update api_key table charset" 2024-03-07T03:09:50.170 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.000121884Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-03-07T03:09:50.170 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.100983797Z level=info msg="Executing migration" id="Add service account foreign key" 2024-03-07T03:09:50.170 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.126342995Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-03-07T03:09:50.170 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.15164481Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-03-07T03:09:50.170 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.176917281Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-03-07T03:09:50.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[21362]: cluster 2024-03-07T03:09:48.983423+0000 mgr.y (mgr.24287) 54 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:50.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[21362]: audit 2024-03-07T03:09:49.050300+0000 mon.a (mon.0) 734 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[21362]: audit 2024-03-07T03:09:49.051765+0000 mon.b (mon.1) 66 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:50.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[21362]: audit 2024-03-07T03:09:49.104168+0000 mon.a (mon.0) 735 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[21362]: audit 2024-03-07T03:09:49.431993+0000 mon.a (mon.0) 736 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[21362]: audit 2024-03-07T03:09:49.441810+0000 mon.a (mon.0) 737 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[31149]: cluster 2024-03-07T03:09:48.983423+0000 mgr.y (mgr.24287) 54 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:50.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[31149]: audit 2024-03-07T03:09:49.050300+0000 mon.a (mon.0) 734 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[31149]: audit 2024-03-07T03:09:49.051765+0000 mon.b (mon.1) 66 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:09:50.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[31149]: audit 2024-03-07T03:09:49.104168+0000 mon.a (mon.0) 735 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[31149]: audit 2024-03-07T03:09:49.431993+0000 mon.a (mon.0) 736 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:50 smithi148 bash[31149]: audit 2024-03-07T03:09:49.441810+0000 mon.a (mon.0) 737 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.202248796Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.244251857Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.294628199Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.336661179Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.362008556Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.387166873Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.412354544Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-03-07T03:09:50.447 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.454683582Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-03-07T03:09:50.708 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.471597626Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-03-07T03:09:50.708 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.488469899Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-03-07T03:09:50.708 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.505378142Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.522225145Z level=info msg="Executing migration" id="create quota table v1" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.56435136Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.589659815Z level=info msg="Executing migration" id="Update quota table charset" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.614781407Z level=info msg="Executing migration" id="create plugin_setting table" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.64012923Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.665300551Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.690584249Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-03-07T03:09:50.709 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.715799512Z level=info msg="Executing migration" id="create session table" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.758021545Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.783106984Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.808405206Z level=info msg="Executing migration" id="create playlist table v2" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.833697287Z level=info msg="Executing migration" id="create playlist item table v2" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.858871168Z level=info msg="Executing migration" id="Update playlist table charset" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.884173765Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.909395835Z level=info msg="Executing migration" id="drop preferences table v2" 2024-03-07T03:09:50.969 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.93465637Z level=info msg="Executing migration" id="drop preferences table v3" 2024-03-07T03:09:51.246 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:50 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:50.976635696Z level=info msg="Executing migration" id="create preferences table v3" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.018654114Z level=info msg="Executing migration" id="Update preferences table charset" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.043816466Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.069033664Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.110991908Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.136273858Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.161552844Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.186752249Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-03-07T03:09:51.247 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.212073877Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-03-07T03:09:51.248 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.254135266Z level=info msg="Executing migration" id="create alert table v1" 2024-03-07T03:09:51.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[31149]: audit 2024-03-07T03:09:50.266377+0000 mon.a (mon.0) 738 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.347 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[31149]: audit 2024-03-07T03:09:50.280928+0000 mon.a (mon.0) 739 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.347 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[31149]: audit 2024-03-07T03:09:50.282763+0000 mon.b (mon.1) 67 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:51.347 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[31149]: audit 2024-03-07T03:09:50.284776+0000 mon.b (mon.1) 68 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:51.347 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[31149]: audit 2024-03-07T03:09:50.297576+0000 mon.a (mon.0) 740 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.347 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[31149]: cephadm 2024-03-07T03:09:50.372767+0000 mgr.y (mgr.24287) 55 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[31149]: cephadm 2024-03-07T03:09:50.382041+0000 mgr.y (mgr.24287) 56 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi148 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[21362]: audit 2024-03-07T03:09:50.266377+0000 mon.a (mon.0) 738 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[21362]: audit 2024-03-07T03:09:50.280928+0000 mon.a (mon.0) 739 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[21362]: audit 2024-03-07T03:09:50.282763+0000 mon.b (mon.1) 67 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[21362]: audit 2024-03-07T03:09:50.284776+0000 mon.b (mon.1) 68 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[21362]: audit 2024-03-07T03:09:50.297576+0000 mon.a (mon.0) 740 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[21362]: cephadm 2024-03-07T03:09:50.372767+0000 mgr.y (mgr.24287) 55 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-03-07T03:09:51.348 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:51 smithi148 bash[21362]: cephadm 2024-03-07T03:09:50.382041+0000 mgr.y (mgr.24287) 56 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi148 2024-03-07T03:09:51.524 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[23221]: audit 2024-03-07T03:09:50.266377+0000 mon.a (mon.0) 738 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.524 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[23221]: audit 2024-03-07T03:09:50.280928+0000 mon.a (mon.0) 739 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.524 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[23221]: audit 2024-03-07T03:09:50.282763+0000 mon.b (mon.1) 67 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:09:51.524 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[23221]: audit 2024-03-07T03:09:50.284776+0000 mon.b (mon.1) 68 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:09:51.524 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[23221]: audit 2024-03-07T03:09:50.297576+0000 mon.a (mon.0) 740 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:51.525 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[23221]: cephadm 2024-03-07T03:09:50.372767+0000 mgr.y (mgr.24287) 55 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-03-07T03:09:51.525 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[23221]: cephadm 2024-03-07T03:09:50.382041+0000 mgr.y (mgr.24287) 56 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi148 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.296347528Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.33004387Z level=info msg="Executing migration" id="add index alert state" 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.34695593Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.36386372Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.397418148Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.439450267Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.489803405Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-03-07T03:09:51.525 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.53172701Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-03-07T03:09:51.793 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.557075565Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-03-07T03:09:51.793 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.599056852Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-03-07T03:09:51.793 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.624282019Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-03-07T03:09:51.794 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.649606868Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-03-07T03:09:51.794 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.674805669Z level=info msg="Executing migration" id="Add column is_default" 2024-03-07T03:09:51.794 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.700027834Z level=info msg="Executing migration" id="Add column frequency" 2024-03-07T03:09:51.794 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.72512298Z level=info msg="Executing migration" id="Add column send_reminder" 2024-03-07T03:09:51.794 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.750465719Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-03-07T03:09:51.794 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.775753721Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-03-07T03:09:51.794 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.801057537Z level=info msg="Executing migration" id="Update alert table charset" 2024-03-07T03:09:51.871 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.826220646Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.851456027Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.893514729Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.918775012Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.944027173Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:51 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:51.985954801Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.011143206Z level=info msg="Executing migration" id="Add for to alert table" 2024-03-07T03:09:52.054 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.036440883Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-03-07T03:09:52.055 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.061682465Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-03-07T03:09:52.306 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[23221]: cluster 2024-03-07T03:09:50.984276+0000 mgr.y (mgr.24287) 57 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:52.307 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.086898544Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-03-07T03:09:52.307 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.128926136Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-03-07T03:09:52.307 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.171023786Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-03-07T03:09:52.307 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.204897933Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-03-07T03:09:52.307 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.230122918Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-03-07T03:09:52.307 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.272102778Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-03-07T03:09:52.307 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.314173731Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-03-07T03:09:52.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:52 smithi148 bash[21362]: cluster 2024-03-07T03:09:50.984276+0000 mgr.y (mgr.24287) 57 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:52.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:52 smithi148 bash[31149]: cluster 2024-03-07T03:09:50.984276+0000 mgr.y (mgr.24287) 57 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.339408487Z level=info msg="Executing migration" id="create annotation table v5" 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.381337734Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.39830967Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.431895346Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.465557998Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.482495444Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.516112549Z level=info msg="Executing migration" id="Update annotation table charset" 2024-03-07T03:09:52.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.558009847Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.600044406Z level=info msg="Executing migration" id="Drop category_id index" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.625255331Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.650554862Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.692687238Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.717869665Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.743107423Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.776649443Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.801882799Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.827101271Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-03-07T03:09:52.845 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.852345683Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-03-07T03:09:53.105 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.894280576Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-03-07T03:09:53.105 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.936293059Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-03-07T03:09:53.105 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:52 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:52.96156722Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-03-07T03:09:53.105 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.003480091Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-03-07T03:09:53.105 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.04557001Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-03-07T03:09:53.105 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.070860781Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-03-07T03:09:53.106 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.112790665Z level=info msg="Executing migration" id="Add epoch_end column" 2024-03-07T03:09:53.366 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.146654996Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-03-07T03:09:53.366 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.171943405Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-03-07T03:09:53.366 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.197072994Z level=info msg="Executing migration" id="Move region to single row" 2024-03-07T03:09:53.366 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.239101684Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-03-07T03:09:53.366 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.281074233Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-03-07T03:09:53.366 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.323066651Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-03-07T03:09:53.367 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.348344847Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-03-07T03:09:53.367 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.37355451Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-03-07T03:09:53.626 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.415531685Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-03-07T03:09:53.626 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.457532189Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-03-07T03:09:53.627 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.482874504Z level=info msg="Executing migration" id="create test_data table" 2024-03-07T03:09:53.627 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.524916888Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-03-07T03:09:53.627 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.566825007Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-03-07T03:09:53.627 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.608815448Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-03-07T03:09:53.627 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.634047242Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-03-07T03:09:53.887 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.659204953Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-03-07T03:09:53.887 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.684506569Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-03-07T03:09:53.887 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.709777231Z level=info msg="Executing migration" id="create team table" 2024-03-07T03:09:53.887 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.735033967Z level=info msg="Executing migration" id="add index team.org_id" 2024-03-07T03:09:53.887 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.760361595Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-03-07T03:09:53.888 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.785521707Z level=info msg="Executing migration" id="create team member table" 2024-03-07T03:09:53.888 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.810771549Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-03-07T03:09:53.888 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.836039909Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-03-07T03:09:53.888 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.894622902Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-03-07T03:09:54.148 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.91154781Z level=info msg="Executing migration" id="Add column email to team table" 2024-03-07T03:09:54.148 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.928391359Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-03-07T03:09:54.148 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.945218416Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-03-07T03:09:54.148 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.978918364Z level=info msg="Executing migration" id="create dashboard acl table" 2024-03-07T03:09:54.149 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:53 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:53.995894205Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-03-07T03:09:54.149 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.021122983Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-03-07T03:09:54.149 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.046666707Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-03-07T03:09:54.149 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.088690553Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-03-07T03:09:54.149 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.130729719Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-03-07T03:09:54.149 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.155871676Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-03-07T03:09:54.417 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[23221]: cluster 2024-03-07T03:09:52.984806+0000 mgr.y (mgr.24287) 58 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:54.417 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.181094857Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-03-07T03:09:54.418 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.206347875Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-03-07T03:09:54.418 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.248419859Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-03-07T03:09:54.418 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.273610785Z level=info msg="Executing migration" id="create tag table" 2024-03-07T03:09:54.418 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.315784642Z level=info msg="Executing migration" id="add index tag.key_value" 2024-03-07T03:09:54.418 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.341044932Z level=info msg="Executing migration" id="create login attempt table" 2024-03-07T03:09:54.418 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.382978767Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-03-07T03:09:54.418 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.4249652Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-03-07T03:09:54.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:54 smithi148 bash[21362]: cluster 2024-03-07T03:09:52.984806+0000 mgr.y (mgr.24287) 58 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:54.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:54 smithi148 bash[31149]: cluster 2024-03-07T03:09:52.984806+0000 mgr.y (mgr.24287) 58 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.458507198Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.500483648Z level=info msg="Executing migration" id="create login_attempt v2" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.517435686Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.551110197Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.568020786Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.593237678Z level=info msg="Executing migration" id="create user auth table" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.618545478Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.643768269Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.668988504Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-03-07T03:09:54.687 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.694264324Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-03-07T03:09:54.956 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.719516344Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-03-07T03:09:54.956 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.744786558Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-03-07T03:09:54.956 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.770030746Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-03-07T03:09:54.956 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.79527924Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-03-07T03:09:54.956 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.820470469Z level=info msg="Executing migration" id="create server_lock table" 2024-03-07T03:09:54.956 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.845797675Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-03-07T03:09:54.956 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.887806362Z level=info msg="Executing migration" id="create user auth token table" 2024-03-07T03:09:54.957 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.91301008Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-03-07T03:09:54.957 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.938523295Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-03-07T03:09:54.957 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:54 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:54.963752256Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-03-07T03:09:55.043 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:09:54 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:09:54] "GET /metrics HTTP/1.1" 200 28557 "" "Prometheus/2.43.0" 2024-03-07T03:09:55.044 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:54 smithi148 systemd[1]: Stopping Ceph alertmanager.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:09:55.207 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.005803793Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-03-07T03:09:55.208 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.031038461Z level=info msg="Executing migration" id="create cache_data table" 2024-03-07T03:09:55.208 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.073112115Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-03-07T03:09:55.208 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.098394427Z level=info msg="Executing migration" id="create short_url table v1" 2024-03-07T03:09:55.208 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.215439387Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-03-07T03:09:55.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.265741275Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-03-07T03:09:55.484 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.357644155Z level=info msg="Executing migration" id="delete alert_definition table" 2024-03-07T03:09:55.485 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:55 smithi148 bash[56354]: ts=2024-03-07T03:09:55.118Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.491390454Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.575425234Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.617265477Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.642614099Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.667765384Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.693031059Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.718306654Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-03-07T03:09:55.736 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.743554766Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-03-07T03:09:56.014 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.768857477Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.794061914Z level=info msg="Executing migration" id="drop alert_definition table" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.819465447Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.836278821Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.853393267Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.886836508Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.903834852Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.920747192Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.946046836Z level=info msg="Executing migration" id="create alert_instance table" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:55 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:55.979621574Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-03-07T03:09:56.015 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.02192993Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-03-07T03:09:56.043 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:55 smithi148 bash[58077]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:09:56.043 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:55 smithi148 bash[58182]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:09:56.043 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:56 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-07T03:09:56.043 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:56 smithi148 systemd[1]: Stopped Ceph alertmanager.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:09:56.044 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:56 smithi148 systemd[1]: Started Ceph alertmanager.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:09:56.234 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:09:56.234 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":64,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:09:39.358876+0000","last_up_change":"2024-03-07T03:07:53.161981+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:09:29.853395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6803","nonce":1285694746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6805","nonce":1285694746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6809","nonce":1285694746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6807","nonce":1285694746}]},"public_addr":"172.21.15.148:6803/1285694746","cluster_addr":"172.21.15.148:6805/1285694746","heartbeat_back_addr":"172.21.15.148:6809/1285694746","heartbeat_front_addr":"172.21.15.148:6807/1285694746","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6811","nonce":3763364013}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6813","nonce":3763364013}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6817","nonce":3763364013}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6815","nonce":3763364013}]},"public_addr":"172.21.15.148:6811/3763364013","cluster_addr":"172.21.15.148:6813/3763364013","heartbeat_back_addr":"172.21.15.148:6817/3763364013","heartbeat_front_addr":"172.21.15.148:6815/3763364013","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6819","nonce":916957575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6821","nonce":916957575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6825","nonce":916957575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6823","nonce":916957575}]},"public_addr":"172.21.15.148:6819/916957575","cluster_addr":"172.21.15.148:6821/916957575","heartbeat_back_addr":"172.21.15.148:6825/916957575","heartbeat_front_addr":"172.21.15.148:6823/916957575","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6827","nonce":2267330328}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6829","nonce":2267330328}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6833","nonce":2267330328}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6831","nonce":2267330328}]},"public_addr":"172.21.15.148:6827/2267330328","cluster_addr":"172.21.15.148:6829/2267330328","heartbeat_back_addr":"172.21.15.148:6833/2267330328","heartbeat_front_addr":"172.21.15.148:6831/2267330328","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6801","nonce":1363408518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6803","nonce":1363408518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6807","nonce":1363408518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6805","nonce":1363408518}]},"public_addr":"172.21.15.195:6801/1363408518","cluster_addr":"172.21.15.195:6803/1363408518","heartbeat_back_addr":"172.21.15.195:6807/1363408518","heartbeat_front_addr":"172.21.15.195:6805/1363408518","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:6801/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/1901736382":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/4143961239":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/3220133019":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2258249874":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.063939325Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.0891069Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.131199549Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.156491664Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.189950961Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.223611409Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.25727556Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-03-07T03:09:56.267 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.274246889Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.299543038Z level=info msg="Executing migration" id="create alert_rule table" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.333292208Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.35860189Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.392133114Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.425880051Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.459608271Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.476584503Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.518583747Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-03-07T03:09:56.553 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[23221]: cluster 2024-03-07T03:09:54.985397+0000 mgr.y (mgr.24287) 59 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:56.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[23221]: audit 2024-03-07T03:09:56.044676+0000 mon.a (mon.0) 741 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:56.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[23221]: audit 2024-03-07T03:09:56.057536+0000 mon.a (mon.0) 742 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:56.554 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[23221]: audit 2024-03-07T03:09:56.242263+0000 mon.a (mon.0) 743 : audit 0 from='client.? 172.21.15.148:0/3740025856' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[21362]: cluster 2024-03-07T03:09:54.985397+0000 mgr.y (mgr.24287) 59 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[21362]: audit 2024-03-07T03:09:56.044676+0000 mon.a (mon.0) 741 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[21362]: audit 2024-03-07T03:09:56.057536+0000 mon.a (mon.0) 742 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[21362]: audit 2024-03-07T03:09:56.242263+0000 mon.a (mon.0) 743 : audit 0 from='client.? 172.21.15.148:0/3740025856' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[31149]: cluster 2024-03-07T03:09:54.985397+0000 mgr.y (mgr.24287) 59 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[31149]: audit 2024-03-07T03:09:56.044676+0000 mon.a (mon.0) 741 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[31149]: audit 2024-03-07T03:09:56.057536+0000 mon.a (mon.0) 742 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:09:56.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:56 smithi148 bash[31149]: audit 2024-03-07T03:09:56.242263+0000 mon.a (mon.0) 743 : audit 0 from='client.? 172.21.15.148:0/3740025856' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:09:56.819 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.560595752Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-03-07T03:09:56.819 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.585815232Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-03-07T03:09:56.819 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.611058665Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-03-07T03:09:56.819 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.652723632Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-03-07T03:09:57.152 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-07T03:09:57.152 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:09:57.303 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:56 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:56.959512846Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-03-07T03:09:57.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[21362]: cephadm 2024-03-07T03:09:56.110561+0000 mgr.y (mgr.24287) 60 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:09:57.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[21362]: cephadm 2024-03-07T03:09:56.369479+0000 mgr.y (mgr.24287) 61 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:09:57.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[31149]: cephadm 2024-03-07T03:09:56.110561+0000 mgr.y (mgr.24287) 60 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:09:57.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[31149]: cephadm 2024-03-07T03:09:56.369479+0000 mgr.y (mgr.24287) 61 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:09:57.544 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.363Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-07T03:09:57.545 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.363Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-07T03:09:57.545 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.365Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.148 port=9094 2024-03-07T03:09:57.545 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.365Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-07T03:09:57.545 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.395Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:09:57.545 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.396Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-07T03:09:57.545 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.399Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-07T03:09:57.545 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:57 smithi148 bash[58229]: ts=2024-03-07T03:09:57.399Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-07T03:09:57.567 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:57 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:57.374963632Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-03-07T03:09:57.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:57 smithi195 bash[23221]: cephadm 2024-03-07T03:09:56.110561+0000 mgr.y (mgr.24287) 60 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:09:57.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:57 smithi195 bash[23221]: cephadm 2024-03-07T03:09:56.369479+0000 mgr.y (mgr.24287) 61 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:09:57.913 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:57 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:57.670253818Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-03-07T03:09:58.227 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:57 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:57.920495686Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-03-07T03:09:58.227 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.041435761Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-03-07T03:09:58.227 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.134094735Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-03-07T03:09:58.528 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[23221]: cluster 2024-03-07T03:09:56.986040+0000 mgr.y (mgr.24287) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:58.529 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.234478723Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-03-07T03:09:58.529 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.343501681Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-03-07T03:09:58.529 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.393600382Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-03-07T03:09:58.529 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.469142564Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-03-07T03:09:58.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:09:58 smithi148 bash[21362]: cluster 2024-03-07T03:09:56.986040+0000 mgr.y (mgr.24287) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:58.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:09:58 smithi148 bash[31149]: cluster 2024-03-07T03:09:56.986040+0000 mgr.y (mgr.24287) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:09:58.817 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.53620786Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-03-07T03:09:58.818 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.578226852Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-03-07T03:09:58.818 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.62055223Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-03-07T03:09:58.818 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.662613627Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-03-07T03:09:58.818 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.73018383Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-03-07T03:09:59.249 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.836350562Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-03-07T03:09:59.249 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:58 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:58.953033908Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-03-07T03:09:59.249 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:59 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:59.077830144Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-03-07T03:09:59.790 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:59 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:59.256406396Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-03-07T03:09:59.790 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:59 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:59.532989355Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-03-07T03:09:59.790 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:59 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:59.608619333Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-03-07T03:09:59.790 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:59 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:59.692757558Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-03-07T03:09:59.790 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:59 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:59.76894083Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-03-07T03:09:59.793 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:09:59 smithi148 bash[58229]: ts=2024-03-07T03:09:59.366Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000395383s 2024-03-07T03:10:00.068 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:09:59 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:09:59.942394684Z level=info msg="Executing migration" id="create provenance_type table" 2024-03-07T03:10:00.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:00 smithi195 bash[23221]: cluster 2024-03-07T03:09:58.986634+0000 mgr.y (mgr.24287) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:00.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:00 smithi195 bash[23221]: cluster 2024-03-07T03:10:00.000188+0000 mon.a (mon.0) 744 : cluster 1 overall HEALTH_OK 2024-03-07T03:10:00.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:00 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:00.232913839Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-03-07T03:10:00.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:00 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:00.398630855Z level=info msg="Executing migration" id="create alert_image table" 2024-03-07T03:10:00.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:00 smithi148 bash[21362]: cluster 2024-03-07T03:09:58.986634+0000 mgr.y (mgr.24287) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:00.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:00 smithi148 bash[21362]: cluster 2024-03-07T03:10:00.000188+0000 mon.a (mon.0) 744 : cluster 1 overall HEALTH_OK 2024-03-07T03:10:00.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:00 smithi148 bash[31149]: cluster 2024-03-07T03:09:58.986634+0000 mgr.y (mgr.24287) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:00.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:00 smithi148 bash[31149]: cluster 2024-03-07T03:10:00.000188+0000 mon.a (mon.0) 744 : cluster 1 overall HEALTH_OK 2024-03-07T03:10:01.017 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:00 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:00.76956617Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-03-07T03:10:01.017 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:00 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:00.915529645Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-03-07T03:10:01.285 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.024752353Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-03-07T03:10:01.285 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.12511001Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-03-07T03:10:01.285 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.200533156Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-03-07T03:10:01.285 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.200959951Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-03-07T03:10:01.567 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.292832702Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-03-07T03:10:01.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.368648307Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-03-07T03:10:01.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.438638505Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-03-07T03:10:01.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.439155997Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-03-07T03:10:01.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.439674799Z level=info msg="alerts found to migrate" alerts=0 2024-03-07T03:10:01.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.477720839Z level=info msg="Executing migration" id="create library_element table v1" 2024-03-07T03:10:01.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.511603993Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-03-07T03:10:01.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.528603042Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-03-07T03:10:01.884 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.579216972Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-03-07T03:10:01.885 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.646295161Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-03-07T03:10:01.885 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.696945569Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-03-07T03:10:01.885 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.765839836Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-03-07T03:10:01.885 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:01 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:01.892374673Z level=info msg="Executing migration" id="create data_keys table" 2024-03-07T03:10:01.890 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:02.217 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.031540206Z level=info msg="Executing migration" id="create secrets table" 2024-03-07T03:10:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[23221]: cluster 2024-03-07T03:10:00.987301+0000 mgr.y (mgr.24287) 64 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:02.562 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.225572721Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-03-07T03:10:02.769 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:02 smithi148 bash[21362]: cluster 2024-03-07T03:10:00.987301+0000 mgr.y (mgr.24287) 64 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:02.769 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:02 smithi148 bash[31149]: cluster 2024-03-07T03:10:00.987301+0000 mgr.y (mgr.24287) 64 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:02.817 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 systemd[1]: Stopping Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:10:02.817 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.569615409Z level=info msg="Executing migration" id="add name column into data_keys" 2024-03-07T03:10:02.817 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.692341498Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-03-07T03:10:02.818 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.767919731Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-03-07T03:10:03.144 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.862451969Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-03-07T03:10:03.145 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.927691938Z level=info msg="Executing migration" id="create kv_store table v1" 2024-03-07T03:10:03.145 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.953160015Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-03-07T03:10:03.145 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:02.986821763Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-03-07T03:10:03.145 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:03 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:03.04598338Z level=info msg="Executing migration" id="create permission table" 2024-03-07T03:10:03.145 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:03 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:03.152360441Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-03-07T03:10:03.145 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:10:03.145 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:10:03.145 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.955Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.956Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.956Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:10:03.146 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:02 smithi195 bash[47245]: ts=2024-03-07T03:10:02.956Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:10:03.506 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:03 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:03.269043728Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-03-07T03:10:03.773 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:03 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:03.513642296Z level=info msg="Executing migration" id="create role table" 2024-03-07T03:10:04.067 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:03 smithi195 bash[49633]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:10:04.068 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:03 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:03.781338533Z level=info msg="Executing migration" id="add column display_name" 2024-03-07T03:10:04.367 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[23221]: cluster 2024-03-07T03:10:02.987740+0000 mgr.y (mgr.24287) 65 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:04.367 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[23221]: audit 2024-03-07T03:10:04.044787+0000 mon.b (mon.1) 69 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:04.367 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[23221]: audit 2024-03-07T03:10:04.307314+0000 mon.a (mon.0) 745 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:04.367 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[23221]: audit 2024-03-07T03:10:04.317504+0000 mon.a (mon.0) 746 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:04.367 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:04.101437835Z level=info msg="Executing migration" id="add column group_name" 2024-03-07T03:10:04.367 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:04.166725569Z level=info msg="Executing migration" id="add index role.org_id" 2024-03-07T03:10:04.367 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:04.242531609Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-03-07T03:10:04.367 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:04.285055432Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-03-07T03:10:04.368 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49694]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:10:04.368 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:04 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-07T03:10:04.368 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:04 smithi195 systemd[1]: Stopped Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:10:04.368 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:04 smithi195 systemd[1]: Started Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:10:04.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21362]: cluster 2024-03-07T03:10:02.987740+0000 mgr.y (mgr.24287) 65 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:04.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21362]: audit 2024-03-07T03:10:04.044787+0000 mon.b (mon.1) 69 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:04.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21362]: audit 2024-03-07T03:10:04.307314+0000 mon.a (mon.0) 745 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:04.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21362]: audit 2024-03-07T03:10:04.317504+0000 mon.a (mon.0) 746 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[31149]: cluster 2024-03-07T03:10:02.987740+0000 mgr.y (mgr.24287) 65 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[31149]: audit 2024-03-07T03:10:04.044787+0000 mon.b (mon.1) 69 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[31149]: audit 2024-03-07T03:10:04.307314+0000 mon.a (mon.0) 745 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[31149]: audit 2024-03-07T03:10:04.317504+0000 mon.a (mon.0) 746 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STOPPING 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STOPPED 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STARTING 2024-03-07T03:10:04.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Serving on http://:::9283 2024-03-07T03:10:04.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STARTED 2024-03-07T03:10:04.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STOPPING 2024-03-07T03:10:04.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:10:04.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STOPPED 2024-03-07T03:10:04.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STARTING 2024-03-07T03:10:04.648 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:04.374748779Z level=info msg="Executing migration" id="create team role table" 2024-03-07T03:10:04.670 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:10:04.670 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":64,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:09:39.358876+0000","last_up_change":"2024-03-07T03:07:53.161981+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:09:29.853395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6803","nonce":1285694746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6805","nonce":1285694746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6809","nonce":1285694746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6807","nonce":1285694746}]},"public_addr":"172.21.15.148:6803/1285694746","cluster_addr":"172.21.15.148:6805/1285694746","heartbeat_back_addr":"172.21.15.148:6809/1285694746","heartbeat_front_addr":"172.21.15.148:6807/1285694746","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6811","nonce":3763364013}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6813","nonce":3763364013}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6817","nonce":3763364013}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6815","nonce":3763364013}]},"public_addr":"172.21.15.148:6811/3763364013","cluster_addr":"172.21.15.148:6813/3763364013","heartbeat_back_addr":"172.21.15.148:6817/3763364013","heartbeat_front_addr":"172.21.15.148:6815/3763364013","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6819","nonce":916957575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6821","nonce":916957575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6825","nonce":916957575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6823","nonce":916957575}]},"public_addr":"172.21.15.148:6819/916957575","cluster_addr":"172.21.15.148:6821/916957575","heartbeat_back_addr":"172.21.15.148:6825/916957575","heartbeat_front_addr":"172.21.15.148:6823/916957575","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6827","nonce":2267330328}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6829","nonce":2267330328}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6833","nonce":2267330328}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6831","nonce":2267330328}]},"public_addr":"172.21.15.148:6827/2267330328","cluster_addr":"172.21.15.148:6829/2267330328","heartbeat_back_addr":"172.21.15.148:6833/2267330328","heartbeat_front_addr":"172.21.15.148:6831/2267330328","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6801","nonce":1363408518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6803","nonce":1363408518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6807","nonce":1363408518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6805","nonce":1363408518}]},"public_addr":"172.21.15.195:6801/1363408518","cluster_addr":"172.21.15.195:6803/1363408518","heartbeat_back_addr":"172.21.15.195:6807/1363408518","heartbeat_front_addr":"172.21.15.195:6805/1363408518","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:6801/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/1901736382":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/4143961239":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/3220133019":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2258249874":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Serving on http://:::9283 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STARTED 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STOPPING 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STOPPED 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STARTING 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Serving on http://:::9283 2024-03-07T03:10:04.900 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:04 smithi148 bash[21615]: [07/Mar/2024:03:10:04] ENGINE Bus STARTED 2024-03-07T03:10:04.901 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:04.65205549Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-03-07T03:10:05.290 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:04 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:04.908121403Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-03-07T03:10:05.310 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-07T03:10:05.310 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-07T03:10:05.311 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-07T03:10:05.311 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-07T03:10:05.311 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-07T03:10:05.312 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-07T03:10:05.312 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-07T03:10:05.313 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-07T03:10:05.567 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.298196946Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-03-07T03:10:05.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.361842816Z level=info msg="Executing migration" id="create user role table" 2024-03-07T03:10:05.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.507155269Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-03-07T03:10:05.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.364632+0000 mon.b (mon.1) 70 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:10:05.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.366681+0000 mon.b (mon.1) 71 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi148.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:10:05.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.367258+0000 mgr.y (mgr.24287) 66 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:10:05.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.369137+0000 mgr.y (mgr.24287) 67 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi148.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:10:05.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.374384+0000 mon.a (mon.0) 747 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.382295+0000 mon.b (mon.1) 72 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.384922+0000 mgr.y (mgr.24287) 68 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.386143+0000 mon.b (mon.1) 73 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.388768+0000 mgr.y (mgr.24287) 69 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.394903+0000 mon.a (mon.0) 748 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.402325+0000 mon.b (mon.1) 74 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.404917+0000 mgr.y (mgr.24287) 70 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.405727+0000 mon.b (mon.1) 75 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.408318+0000 mgr.y (mgr.24287) 71 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.414544+0000 mon.a (mon.0) 749 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.425426+0000 mon.b (mon.1) 76 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:10:05.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[23221]: audit 2024-03-07T03:10:04.678919+0000 mon.a (mon.0) 750 : audit 0 from='client.? 172.21.15.148:0/1236085752' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:10:05.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.364632+0000 mon.b (mon.1) 70 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:10:05.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.366681+0000 mon.b (mon.1) 71 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi148.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:10:05.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.367258+0000 mgr.y (mgr.24287) 66 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:10:05.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.369137+0000 mgr.y (mgr.24287) 67 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi148.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:10:05.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.374384+0000 mon.a (mon.0) 747 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.382295+0000 mon.b (mon.1) 72 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.384922+0000 mgr.y (mgr.24287) 68 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.386143+0000 mon.b (mon.1) 73 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.388768+0000 mgr.y (mgr.24287) 69 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.394903+0000 mon.a (mon.0) 748 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.402325+0000 mon.b (mon.1) 74 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.404917+0000 mgr.y (mgr.24287) 70 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.405727+0000 mon.b (mon.1) 75 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.408318+0000 mgr.y (mgr.24287) 71 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.414544+0000 mon.a (mon.0) 749 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.425426+0000 mon.b (mon.1) 76 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[21362]: audit 2024-03-07T03:10:04.678919+0000 mon.a (mon.0) 750 : audit 0 from='client.? 172.21.15.148:0/1236085752' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:10:05.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.364632+0000 mon.b (mon.1) 70 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.366681+0000 mon.b (mon.1) 71 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi148.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.367258+0000 mgr.y (mgr.24287) 66 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.369137+0000 mgr.y (mgr.24287) 67 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi148.front.sepia.ceph.com:9093"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.374384+0000 mon.a (mon.0) 747 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.382295+0000 mon.b (mon.1) 72 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.384922+0000 mgr.y (mgr.24287) 68 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.386143+0000 mon.b (mon.1) 73 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.388768+0000 mgr.y (mgr.24287) 69 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi195.front.sepia.ceph.com:9095"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.394903+0000 mon.a (mon.0) 748 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.402325+0000 mon.b (mon.1) 74 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.404917+0000 mgr.y (mgr.24287) 70 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.405727+0000 mon.b (mon.1) 75 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:10:05.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.408318+0000 mgr.y (mgr.24287) 71 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi195.front.sepia.ceph.com:3000"}]: dispatch 2024-03-07T03:10:05.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.414544+0000 mon.a (mon.0) 749 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:05.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.425426+0000 mon.b (mon.1) 76 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:10:05.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:05 smithi148 bash[31149]: audit 2024-03-07T03:10:04.678919+0000 mon.a (mon.0) 750 : audit 0 from='client.? 172.21.15.148:0/1236085752' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:10:05.843 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.599646398Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-03-07T03:10:05.843 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.733501382Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-03-07T03:10:06.124 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.850750307Z level=info msg="Executing migration" id="create builtin role table" 2024-03-07T03:10:06.124 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.892750783Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-03-07T03:10:06.124 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:05 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:05.971300091Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-03-07T03:10:06.125 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.03492397Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-03-07T03:10:06.125 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.085484834Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-03-07T03:10:06.405 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.132452607Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-03-07T03:10:06.405 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.203583654Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-03-07T03:10:06.405 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.220526634Z level=info msg="Executing migration" id="add unique index role.uid" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.237439242Z level=info msg="Executing migration" id="create seed assignment table" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.254293008Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.279708375Z level=info msg="Executing migration" id="add column hidden to role table" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.330227278Z level=info msg="Executing migration" id="create query_history table v1" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.372372864Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.145Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.145Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.145Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi195 (none))" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.145Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:10:06.406 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.145Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.147Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.148Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.149Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.149Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.151Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.151Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.899µs 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.151Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.153Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.153Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.153Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.014µs wal_replay_duration=1.884482ms wbl_replay_duration=184ns total_replay_duration=1.939455ms 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.154Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.154Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.154Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:10:06.407 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.175Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.507158ms db_storage=983ns remote_storage=1.646µs web_handler=312ns query_engine=705ns scrape=269.956µs scrape_sd=88.557µs notify=22.219µs notify_sd=19.656µs rules=19.777625ms tracing=6.328µs 2024-03-07T03:10:06.408 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.175Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:10:06.408 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49720]: ts=2024-03-07T03:10:06.175Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:10:06.668 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[23221]: cluster 2024-03-07T03:10:04.988497+0000 mgr.y (mgr.24287) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.414398456Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.448094249Z level=info msg="Executing migration" id="teams permissions migration" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.465041244Z level=info msg="Executing migration" id="dashboard permissions" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.481996383Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.498906585Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.540963009Z level=info msg="Executing migration" id="alerting notification permissions" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.566283087Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.608307955Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.633512945Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-03-07T03:10:06.668 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.675511248Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-03-07T03:10:06.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:06 smithi148 bash[21362]: cluster 2024-03-07T03:10:04.988497+0000 mgr.y (mgr.24287) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:06.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:06 smithi148 bash[31149]: cluster 2024-03-07T03:10:04.988497+0000 mgr.y (mgr.24287) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.700829095Z level=info msg="Executing migration" id="create correlation table v1" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.745441588Z level=info msg="Executing migration" id="add index correlations.uid" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.76813299Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.793357986Z level=info msg="Executing migration" id="add correlation config column" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.818603525Z level=info msg="Executing migration" id="create entity_events table" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.843902083Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.885969698Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.886343873Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.911187464Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.91156665Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-07T03:10:06.929 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.9364359Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-03-07T03:10:07.190 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:06 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:06.961775368Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-03-07T03:10:07.190 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.003672806Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-03-07T03:10:07.191 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.029076026Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-07T03:10:07.191 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.054306877Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-03-07T03:10:07.191 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.096182653Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-07T03:10:07.191 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.121496209Z level=info msg="Executing migration" id="Drop public config table" 2024-03-07T03:10:07.191 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.146827282Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-03-07T03:10:07.191 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.172351147Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-03-07T03:10:07.191 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.197594587Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-07T03:10:07.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.22292514Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-03-07T03:10:07.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.248146731Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-03-07T03:10:07.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.281685454Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-03-07T03:10:07.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.307016576Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-03-07T03:10:07.459 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.332175978Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-03-07T03:10:07.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.357470428Z level=info msg="Executing migration" id="add share column" 2024-03-07T03:10:07.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.399494438Z level=info msg="Executing migration" id="create default alerting folders" 2024-03-07T03:10:07.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.441567067Z level=info msg="Executing migration" id="create file table" 2024-03-07T03:10:07.460 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.466913742Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.492173727Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.534166194Z level=info msg="Executing migration" id="create file_meta table" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.576125179Z level=info msg="Executing migration" id="file table idx: path key" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.609733956Z level=info msg="Executing migration" id="set path collation in file table" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.626677192Z level=info msg="Executing migration" id="managed permissions migration" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.643631081Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.660490009Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.702557495Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-03-07T03:10:07.737 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.744661634Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-03-07T03:10:07.793 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:10:07 smithi148 bash[58229]: ts=2024-03-07T03:10:07.369Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003845103s 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.769876858Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.803558851Z level=info msg="Executing migration" id="update group index for alert rules" 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.83707955Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.88750312Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.912939877Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.929797394Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.946665312Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-03-07T03:10:08.048 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:07 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:07.988945543Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.055796989Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.081110356Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.123252832Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.165247389Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.190405585Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.232462213Z level=info msg="Executing migration" id="create folder table" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.274279356Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.299685752Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-03-07T03:10:08.317 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.324778862Z level=info msg="Executing migration" id="Update folder title length" 2024-03-07T03:10:08.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[23221]: cluster 2024-03-07T03:10:06.989139+0000 mgr.y (mgr.24287) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:08.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.350089784Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-03-07T03:10:08.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=migrator t=2024-03-07T03:10:08.392244658Z level=info msg="migrations completed" performed=485 skipped=0 duration=22.697768944s 2024-03-07T03:10:08.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=sqlstore t=2024-03-07T03:10:08.392935717Z level=info msg="Created default organization" 2024-03-07T03:10:08.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=plugin.loader t=2024-03-07T03:10:08.549877841Z level=info msg="Plugin registered" pluginID=input 2024-03-07T03:10:08.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:08 smithi148 bash[21362]: cluster 2024-03-07T03:10:06.989139+0000 mgr.y (mgr.24287) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:08.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:08 smithi148 bash[31149]: cluster 2024-03-07T03:10:06.989139+0000 mgr.y (mgr.24287) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:09.067 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=plugin.loader t=2024-03-07T03:10:08.581764471Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-03-07T03:10:09.068 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=plugin.loader t=2024-03-07T03:10:08.581799905Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-03-07T03:10:09.068 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=secrets t=2024-03-07T03:10:08.581839858Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-03-07T03:10:09.068 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=query_data t=2024-03-07T03:10:08.583874597Z level=info msg="Query Service initialization" 2024-03-07T03:10:09.068 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:08 smithi195 bash[49044]: logger=live.push_http t=2024-03-07T03:10:08.588161903Z level=info msg="Live Push Gateway initialization" 2024-03-07T03:10:09.283 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:09.283 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:09.283 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:09.283 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:09.285 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:09.286 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:09.286 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:09.289 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:10.115 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:09 smithi195 bash[49044]: logger=infra.usagestats.collector t=2024-03-07T03:10:09.872250318Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-03-07T03:10:10.115 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:09 smithi195 bash[49044]: logger=provisioning.datasources t=2024-03-07T03:10:09.928301438Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-03-07T03:10:10.115 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=provisioning.datasources t=2024-03-07T03:10:10.029282509Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-03-07T03:10:10.115 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=provisioning.alerting t=2024-03-07T03:10:10.111589791Z level=info msg="starting to provision alerting" 2024-03-07T03:10:10.115 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=provisioning.alerting t=2024-03-07T03:10:10.11165248Z level=info msg="finished to provision alerting" 2024-03-07T03:10:10.115 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=grafanaStorageLogger t=2024-03-07T03:10:10.112082297Z level=info msg="storage starting" 2024-03-07T03:10:10.116 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=ngalert.state.manager t=2024-03-07T03:10:10.11197145Z level=info msg="Warming state cache for startup" 2024-03-07T03:10:10.116 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=http.server t=2024-03-07T03:10:10.115494269Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-03-07T03:10:10.116 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=sqlstore.transactions t=2024-03-07T03:10:10.123239208Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-07T03:10:10.567 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=ngalert.state.manager t=2024-03-07T03:10:10.15672982Z level=info msg="State cache has been initialized" states=0 duration=44.757722ms 2024-03-07T03:10:10.567 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=ngalert.multiorg.alertmanager t=2024-03-07T03:10:10.156800427Z level=info msg="starting MultiOrg Alertmanager" 2024-03-07T03:10:10.567 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=ticker t=2024-03-07T03:10:10.156836464Z level=info msg=starting first_tick=2024-03-07T03:10:20Z 2024-03-07T03:10:10.568 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[49044]: logger=sqlstore.transactions t=2024-03-07T03:10:10.280100081Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-07T03:10:11.114 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:10 smithi148 bash[31149]: cluster 2024-03-07T03:10:08.989735+0000 mgr.y (mgr.24287) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:11.114 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:10 smithi148 bash[31149]: audit 2024-03-07T03:10:09.864702+0000 mon.a (mon.0) 751 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:11.114 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:10 smithi148 bash[31149]: audit 2024-03-07T03:10:09.875418+0000 mon.a (mon.0) 752 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:11.114 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:10 smithi148 bash[21362]: cluster 2024-03-07T03:10:08.989735+0000 mgr.y (mgr.24287) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:11.114 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:10 smithi148 bash[21362]: audit 2024-03-07T03:10:09.864702+0000 mon.a (mon.0) 751 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:11.114 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:10 smithi148 bash[21362]: audit 2024-03-07T03:10:09.875418+0000 mon.a (mon.0) 752 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:11.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[23221]: cluster 2024-03-07T03:10:08.989735+0000 mgr.y (mgr.24287) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:11.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[23221]: audit 2024-03-07T03:10:09.864702+0000 mon.a (mon.0) 751 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:11.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:10 smithi195 bash[23221]: audit 2024-03-07T03:10:09.875418+0000 mon.a (mon.0) 752 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.183 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[31149]: cluster 2024-03-07T03:10:10.990417+0000 mgr.y (mgr.24287) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:12.183 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[31149]: audit 2024-03-07T03:10:11.121329+0000 mon.a (mon.0) 753 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.183 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[31149]: audit 2024-03-07T03:10:11.129375+0000 mon.a (mon.0) 754 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.183 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[31149]: audit 2024-03-07T03:10:11.657301+0000 mon.b (mon.1) 77 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:10:12.183 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[31149]: audit 2024-03-07T03:10:11.658832+0000 mon.b (mon.1) 78 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:10:12.184 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[31149]: audit 2024-03-07T03:10:11.667664+0000 mon.a (mon.0) 755 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.184 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[21362]: cluster 2024-03-07T03:10:10.990417+0000 mgr.y (mgr.24287) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:12.184 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[21362]: audit 2024-03-07T03:10:11.121329+0000 mon.a (mon.0) 753 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.184 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[21362]: audit 2024-03-07T03:10:11.129375+0000 mon.a (mon.0) 754 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.184 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[21362]: audit 2024-03-07T03:10:11.657301+0000 mon.b (mon.1) 77 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:10:12.184 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[21362]: audit 2024-03-07T03:10:11.658832+0000 mon.b (mon.1) 78 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:10:12.184 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:12 smithi148 bash[21362]: audit 2024-03-07T03:10:11.667664+0000 mon.a (mon.0) 755 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:12 smithi195 bash[23221]: cluster 2024-03-07T03:10:10.990417+0000 mgr.y (mgr.24287) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:12 smithi195 bash[23221]: audit 2024-03-07T03:10:11.121329+0000 mon.a (mon.0) 753 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:12 smithi195 bash[23221]: audit 2024-03-07T03:10:11.129375+0000 mon.a (mon.0) 754 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:12 smithi195 bash[23221]: audit 2024-03-07T03:10:11.657301+0000 mon.b (mon.1) 77 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:12 smithi195 bash[23221]: audit 2024-03-07T03:10:11.658832+0000 mon.b (mon.1) 78 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:10:12.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:12 smithi195 bash[23221]: audit 2024-03-07T03:10:11.667664+0000 mon.a (mon.0) 755 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:10:14.458 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:14 smithi148 bash[21362]: cluster 2024-03-07T03:10:12.991018+0000 mgr.y (mgr.24287) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:14.459 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:14 smithi148 bash[31149]: cluster 2024-03-07T03:10:12.991018+0000 mgr.y (mgr.24287) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:14.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:14 smithi195 bash[23221]: cluster 2024-03-07T03:10:12.991018+0000 mgr.y (mgr.24287) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:14.749 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:14 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:10:14] "GET /metrics HTTP/1.1" 200 28558 "" "Prometheus/2.43.0" 2024-03-07T03:10:16.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:16 smithi148 bash[31149]: cluster 2024-03-07T03:10:14.991576+0000 mgr.y (mgr.24287) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:16.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:16 smithi148 bash[21362]: cluster 2024-03-07T03:10:14.991576+0000 mgr.y (mgr.24287) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:16.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:16 smithi195 bash[23221]: cluster 2024-03-07T03:10:14.991576+0000 mgr.y (mgr.24287) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:18.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:18 smithi148 bash[21362]: cluster 2024-03-07T03:10:16.992224+0000 mgr.y (mgr.24287) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:18.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:18 smithi148 bash[31149]: cluster 2024-03-07T03:10:16.992224+0000 mgr.y (mgr.24287) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:18 smithi195 bash[23221]: cluster 2024-03-07T03:10:16.992224+0000 mgr.y (mgr.24287) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:19.464 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:19 smithi148 bash[21362]: audit 2024-03-07T03:10:19.044496+0000 mon.b (mon.1) 79 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:19.464 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:19 smithi148 bash[31149]: audit 2024-03-07T03:10:19.044496+0000 mon.b (mon.1) 79 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:19.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:19 smithi195 bash[23221]: audit 2024-03-07T03:10:19.044496+0000 mon.b (mon.1) 79 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:20.355 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:20 smithi148 bash[21362]: cluster 2024-03-07T03:10:18.992762+0000 mgr.y (mgr.24287) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:20.355 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:20 smithi148 bash[31149]: cluster 2024-03-07T03:10:18.992762+0000 mgr.y (mgr.24287) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:20.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:20 smithi195 bash[23221]: cluster 2024-03-07T03:10:18.992762+0000 mgr.y (mgr.24287) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:22.000 INFO:teuthology.orchestra.run.smithi148.stdout:236223201310 2024-03-07T03:10:22.000 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-07T03:10:22.254 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:22 smithi148 bash[21362]: cluster 2024-03-07T03:10:20.993370+0000 mgr.y (mgr.24287) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:22.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:22 smithi148 bash[31149]: cluster 2024-03-07T03:10:20.993370+0000 mgr.y (mgr.24287) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:22.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:22 smithi195 bash[23221]: cluster 2024-03-07T03:10:20.993370+0000 mgr.y (mgr.24287) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:22.871 INFO:teuthology.orchestra.run.smithi148.stdout:55834574938 2024-03-07T03:10:22.872 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-07T03:10:24.263 INFO:teuthology.orchestra.run.smithi148.stdout:111669149768 2024-03-07T03:10:24.263 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-07T03:10:24.264 INFO:teuthology.orchestra.run.smithi148.stdout:201863462954 2024-03-07T03:10:24.264 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-07T03:10:24.490 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:24 smithi148 bash[21362]: cluster 2024-03-07T03:10:22.993905+0000 mgr.y (mgr.24287) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:24.491 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:24 smithi148 bash[31149]: cluster 2024-03-07T03:10:22.993905+0000 mgr.y (mgr.24287) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:24.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:24 smithi195 bash[23221]: cluster 2024-03-07T03:10:22.993905+0000 mgr.y (mgr.24287) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:24.687 INFO:teuthology.orchestra.run.smithi148.stdout:167503724596 2024-03-07T03:10:24.687 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-07T03:10:24.793 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:24 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:10:24] "GET /metrics HTTP/1.1" 200 28557 "" "Prometheus/2.43.0" 2024-03-07T03:10:25.764 INFO:teuthology.orchestra.run.smithi148.stdout:34359738469 2024-03-07T03:10:25.765 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-07T03:10:25.807 INFO:teuthology.orchestra.run.smithi148.stdout:133143986237 2024-03-07T03:10:25.807 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-07T03:10:26.261 INFO:teuthology.orchestra.run.smithi148.stdout:77309411410 2024-03-07T03:10:26.262 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-07T03:10:26.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:26 smithi148 bash[21362]: cluster 2024-03-07T03:10:24.994573+0000 mgr.y (mgr.24287) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:26 smithi148 bash[31149]: cluster 2024-03-07T03:10:24.994573+0000 mgr.y (mgr.24287) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:26.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:26 smithi195 bash[23221]: cluster 2024-03-07T03:10:24.994573+0000 mgr.y (mgr.24287) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:28.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:28 smithi148 bash[21362]: cluster 2024-03-07T03:10:26.995304+0000 mgr.y (mgr.24287) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:28.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:28 smithi148 bash[31149]: cluster 2024-03-07T03:10:26.995304+0000 mgr.y (mgr.24287) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:28.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:28 smithi195 bash[23221]: cluster 2024-03-07T03:10:26.995304+0000 mgr.y (mgr.24287) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:28.778 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:28.779 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:28.779 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:28.781 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:28.781 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:29.794 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:29.794 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:30.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:30 smithi195 bash[23221]: cluster 2024-03-07T03:10:28.996070+0000 mgr.y (mgr.24287) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:30.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:30 smithi148 bash[31149]: cluster 2024-03-07T03:10:28.996070+0000 mgr.y (mgr.24287) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:30 smithi148 bash[21362]: cluster 2024-03-07T03:10:28.996070+0000 mgr.y (mgr.24287) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:30.813 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:32.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:32 smithi148 bash[31149]: cluster 2024-03-07T03:10:30.996865+0000 mgr.y (mgr.24287) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:32.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:32 smithi148 bash[21362]: cluster 2024-03-07T03:10:30.996865+0000 mgr.y (mgr.24287) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:32.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:32 smithi195 bash[23221]: cluster 2024-03-07T03:10:30.996865+0000 mgr.y (mgr.24287) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:34.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:34 smithi148 bash[21362]: cluster 2024-03-07T03:10:32.997367+0000 mgr.y (mgr.24287) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:34.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:34 smithi148 bash[21362]: audit 2024-03-07T03:10:34.045245+0000 mon.b (mon.1) 80 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:34 smithi148 bash[31149]: cluster 2024-03-07T03:10:32.997367+0000 mgr.y (mgr.24287) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:34.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:34 smithi148 bash[31149]: audit 2024-03-07T03:10:34.045245+0000 mon.b (mon.1) 80 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:34.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:34 smithi195 bash[23221]: cluster 2024-03-07T03:10:32.997367+0000 mgr.y (mgr.24287) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:34.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:34 smithi195 bash[23221]: audit 2024-03-07T03:10:34.045245+0000 mon.b (mon.1) 80 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:35.043 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:34 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:10:34] "GET /metrics HTTP/1.1" 200 28549 "" "Prometheus/2.43.0" 2024-03-07T03:10:36.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:36 smithi148 bash[21362]: cluster 2024-03-07T03:10:34.997932+0000 mgr.y (mgr.24287) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:36.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:36 smithi148 bash[31149]: cluster 2024-03-07T03:10:34.997932+0000 mgr.y (mgr.24287) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:36.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:36 smithi195 bash[23221]: cluster 2024-03-07T03:10:34.997932+0000 mgr.y (mgr.24287) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:38.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:38 smithi148 bash[21362]: cluster 2024-03-07T03:10:36.998526+0000 mgr.y (mgr.24287) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:38.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:38 smithi148 bash[31149]: cluster 2024-03-07T03:10:36.998526+0000 mgr.y (mgr.24287) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:38.567 INFO:teuthology.orchestra.run.smithi148.stdout:167503724599 2024-03-07T03:10:38.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:38 smithi195 bash[23221]: cluster 2024-03-07T03:10:36.998526+0000 mgr.y (mgr.24287) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:39.099 INFO:teuthology.orchestra.run.smithi148.stdout:236223201314 2024-03-07T03:10:39.273 INFO:teuthology.orchestra.run.smithi148.stdout:34359738472 2024-03-07T03:10:39.448 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:39 smithi148 bash[31149]: audit 2024-03-07T03:10:38.577790+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.148:0/3653171429' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:10:39.448 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:39 smithi148 bash[31149]: audit 2024-03-07T03:10:39.109769+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.148:0/2674874909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:10:39.448 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:39 smithi148 bash[21362]: audit 2024-03-07T03:10:38.577790+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.148:0/3653171429' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:10:39.448 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:39 smithi148 bash[21362]: audit 2024-03-07T03:10:39.109769+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.148:0/2674874909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:10:39.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:39 smithi195 bash[23221]: audit 2024-03-07T03:10:38.577790+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.148:0/3653171429' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:10:39.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:39 smithi195 bash[23221]: audit 2024-03-07T03:10:39.109769+0000 mon.a (mon.0) 757 : audit 0 from='client.? 172.21.15.148:0/2674874909' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:10:39.768 INFO:teuthology.orchestra.run.smithi148.stdout:55834574942 2024-03-07T03:10:40.273 INFO:teuthology.orchestra.run.smithi148.stdout:201863462957 2024-03-07T03:10:40.307 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724596 got 167503724599 for osd.5 2024-03-07T03:10:40.307 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:40.555 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[21362]: cluster 2024-03-07T03:10:38.998900+0000 mgr.y (mgr.24287) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:40.555 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[21362]: audit 2024-03-07T03:10:39.284304+0000 mon.a (mon.0) 758 : audit 0 from='client.? 172.21.15.148:0/4291189905' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:10:40.556 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[21362]: audit 2024-03-07T03:10:39.779156+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.148:0/4030835779' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:10:40.556 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[21362]: audit 2024-03-07T03:10:40.283920+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.148:0/2902209748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:10:40.556 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[31149]: cluster 2024-03-07T03:10:38.998900+0000 mgr.y (mgr.24287) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:40.556 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[31149]: audit 2024-03-07T03:10:39.284304+0000 mon.a (mon.0) 758 : audit 0 from='client.? 172.21.15.148:0/4291189905' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:10:40.556 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[31149]: audit 2024-03-07T03:10:39.779156+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.148:0/4030835779' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:10:40.556 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:40 smithi148 bash[31149]: audit 2024-03-07T03:10:40.283920+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.148:0/2902209748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:10:40.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:40 smithi195 bash[23221]: cluster 2024-03-07T03:10:38.998900+0000 mgr.y (mgr.24287) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:40.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:40 smithi195 bash[23221]: audit 2024-03-07T03:10:39.284304+0000 mon.a (mon.0) 758 : audit 0 from='client.? 172.21.15.148:0/4291189905' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:10:40.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:40 smithi195 bash[23221]: audit 2024-03-07T03:10:39.779156+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.148:0/4030835779' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:10:40.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:40 smithi195 bash[23221]: audit 2024-03-07T03:10:40.283920+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.148:0/2902209748' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:10:40.862 INFO:teuthology.orchestra.run.smithi148.stdout:133143986240 2024-03-07T03:10:40.985 INFO:teuthology.orchestra.run.smithi148.stdout:111669149771 2024-03-07T03:10:41.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:41 smithi148 bash[21362]: audit 2024-03-07T03:10:40.873513+0000 mon.a (mon.0) 759 : audit 0 from='client.? 172.21.15.148:0/3658972116' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:10:41.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:41 smithi148 bash[21362]: audit 2024-03-07T03:10:40.995840+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.148:0/2466865098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:10:41.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:41 smithi148 bash[31149]: audit 2024-03-07T03:10:40.873513+0000 mon.a (mon.0) 759 : audit 0 from='client.? 172.21.15.148:0/3658972116' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:10:41.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:41 smithi148 bash[31149]: audit 2024-03-07T03:10:40.995840+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.148:0/2466865098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:10:41.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:41 smithi195 bash[23221]: audit 2024-03-07T03:10:40.873513+0000 mon.a (mon.0) 759 : audit 0 from='client.? 172.21.15.148:0/3658972116' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:10:41.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:41 smithi195 bash[23221]: audit 2024-03-07T03:10:40.995840+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.148:0/2466865098' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:10:42.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738469 got 34359738472 for osd.0 2024-03-07T03:10:42.344 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:42.346 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201310 got 236223201314 for osd.7 2024-03-07T03:10:42.346 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:42.363 INFO:teuthology.orchestra.run.smithi148.stdout:77309411413 2024-03-07T03:10:42.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:42 smithi195 bash[23221]: cluster 2024-03-07T03:10:40.999452+0000 mgr.y (mgr.24287) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:42.568 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574938 got 55834574942 for osd.1 2024-03-07T03:10:42.568 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:42.668 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:42 smithi148 bash[21362]: cluster 2024-03-07T03:10:40.999452+0000 mgr.y (mgr.24287) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:42.668 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:42 smithi148 bash[31149]: cluster 2024-03-07T03:10:40.999452+0000 mgr.y (mgr.24287) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:43.246 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462954 got 201863462957 for osd.6 2024-03-07T03:10:43.246 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:43.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:43 smithi148 bash[21362]: audit 2024-03-07T03:10:42.374032+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.148:0/2038096631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:10:43.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:43 smithi148 bash[31149]: audit 2024-03-07T03:10:42.374032+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.148:0/2038096631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:10:43.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:43 smithi195 bash[23221]: audit 2024-03-07T03:10:42.374032+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.148:0/2038096631' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:10:43.768 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986237 got 133143986240 for osd.4 2024-03-07T03:10:43.768 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:44.184 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149768 got 111669149771 for osd.3 2024-03-07T03:10:44.184 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:44.201 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411410 got 77309411413 for osd.2 2024-03-07T03:10:44.202 DEBUG:teuthology.parallel:result is None 2024-03-07T03:10:44.202 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-07T03:10:44.202 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:10:44.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:44 smithi148 bash[21362]: cluster 2024-03-07T03:10:43.000011+0000 mgr.y (mgr.24287) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:44.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:44 smithi148 bash[31149]: cluster 2024-03-07T03:10:43.000011+0000 mgr.y (mgr.24287) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:44.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:44 smithi195 bash[23221]: cluster 2024-03-07T03:10:43.000011+0000 mgr.y (mgr.24287) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:45.043 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:44 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:10:44] "GET /metrics HTTP/1.1" 200 28549 "" "Prometheus/2.43.0" 2024-03-07T03:10:46.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:46 smithi195 bash[23221]: cluster 2024-03-07T03:10:45.000608+0000 mgr.y (mgr.24287) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:46.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:46 smithi148 bash[21362]: cluster 2024-03-07T03:10:45.000608+0000 mgr.y (mgr.24287) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:46.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:46 smithi148 bash[31149]: cluster 2024-03-07T03:10:45.000608+0000 mgr.y (mgr.24287) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:48.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:48 smithi195 bash[23221]: cluster 2024-03-07T03:10:47.000995+0000 mgr.y (mgr.24287) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:48.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:48 smithi148 bash[21362]: cluster 2024-03-07T03:10:47.000995+0000 mgr.y (mgr.24287) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:48.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:48 smithi148 bash[31149]: cluster 2024-03-07T03:10:47.000995+0000 mgr.y (mgr.24287) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:48.974 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:49.745 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:49 smithi148 bash[21362]: audit 2024-03-07T03:10:49.045480+0000 mon.b (mon.1) 81 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:49.745 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:49 smithi148 bash[31149]: audit 2024-03-07T03:10:49.045480+0000 mon.b (mon.1) 81 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:49.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:49 smithi195 bash[23221]: audit 2024-03-07T03:10:49.045480+0000 mon.b (mon.1) 81 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:10:50.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:50 smithi148 bash[21362]: cluster 2024-03-07T03:10:49.001319+0000 mgr.y (mgr.24287) 94 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:50.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:50 smithi148 bash[31149]: cluster 2024-03-07T03:10:49.001319+0000 mgr.y (mgr.24287) 94 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:50.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:50 smithi195 bash[23221]: cluster 2024-03-07T03:10:49.001319+0000 mgr.y (mgr.24287) 94 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:51.671 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:10:51.674 INFO:teuthology.orchestra.run.smithi148.stderr:dumped all 2024-03-07T03:10:52.318 INFO:teuthology.orchestra.run.smithi148.stdout:{"pg_ready":true,"pg_map":{"version":68,"stamp":"2024-03-07T03:10:51.001563+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":222904,"kb_used_data":6744,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749738312,"statfs":{"total":767960285184,"available":767732031488,"internally_reserved":0,"allocated":6905856,"data_stored":3904077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002800"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.989452+0000","last_change":"2024-03-07T03:09:31.339012+0000","last_active":"2024-03-07T03:09:34.989452+0000","last_peered":"2024-03-07T03:09:34.989452+0000","last_clean":"2024-03-07T03:09:34.989452+0000","last_became_active":"2024-03-07T03:09:31.338487+0000","last_became_peered":"2024-03-07T03:09:31.338487+0000","last_unstale":"2024-03-07T03:09:34.989452+0000","last_undegraded":"2024-03-07T03:09:34.989452+0000","last_fullsized":"2024-03-07T03:09:34.989452+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:48:57.153468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.981001+0000","last_change":"2024-03-07T03:09:31.492237+0000","last_active":"2024-03-07T03:09:34.981001+0000","last_peered":"2024-03-07T03:09:34.981001+0000","last_clean":"2024-03-07T03:09:34.981001+0000","last_became_active":"2024-03-07T03:09:31.490659+0000","last_became_peered":"2024-03-07T03:09:31.490659+0000","last_unstale":"2024-03-07T03:09:34.981001+0000","last_undegraded":"2024-03-07T03:09:34.981001+0000","last_fullsized":"2024-03-07T03:09:34.981001+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:58:14.548445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.983783+0000","last_change":"2024-03-07T03:09:31.490148+0000","last_active":"2024-03-07T03:09:34.983783+0000","last_peered":"2024-03-07T03:09:34.983783+0000","last_clean":"2024-03-07T03:09:34.983783+0000","last_became_active":"2024-03-07T03:09:31.489708+0000","last_became_peered":"2024-03-07T03:09:31.489708+0000","last_unstale":"2024-03-07T03:09:34.983783+0000","last_undegraded":"2024-03-07T03:09:34.983783+0000","last_fullsized":"2024-03-07T03:09:34.983783+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:21:06.820876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.983692+0000","last_change":"2024-03-07T03:09:31.490070+0000","last_active":"2024-03-07T03:09:34.983692+0000","last_peered":"2024-03-07T03:09:34.983692+0000","last_clean":"2024-03-07T03:09:34.983692+0000","last_became_active":"2024-03-07T03:09:31.489632+0000","last_became_peered":"2024-03-07T03:09:31.489632+0000","last_unstale":"2024-03-07T03:09:34.983692+0000","last_undegraded":"2024-03-07T03:09:34.983692+0000","last_fullsized":"2024-03-07T03:09:34.983692+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:08:00.120410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":260,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-03-07T03:09:39.563927+0000","last_change":"2024-03-07T03:07:57.209434+0000","last_active":"2024-03-07T03:09:39.563927+0000","last_peered":"2024-03-07T03:09:39.563927+0000","last_clean":"2024-03-07T03:09:39.563927+0000","last_became_active":"2024-03-07T03:07:57.208461+0000","last_became_peered":"2024-03-07T03:07:57.208461+0000","last_unstale":"2024-03-07T03:09:39.563927+0000","last_undegraded":"2024-03-07T03:09:39.563927+0000","last_fullsized":"2024-03-07T03:09:39.563927+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_clean_scrub_stamp":"2024-03-07T03:03:44.169894+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:55:26.915133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.981638+0000","last_change":"2024-03-07T03:09:31.338124+0000","last_active":"2024-03-07T03:09:34.981638+0000","last_peered":"2024-03-07T03:09:34.981638+0000","last_clean":"2024-03-07T03:09:34.981638+0000","last_became_active":"2024-03-07T03:09:31.337531+0000","last_became_peered":"2024-03-07T03:09:31.337531+0000","last_unstale":"2024-03-07T03:09:34.981638+0000","last_undegraded":"2024-03-07T03:09:34.981638+0000","last_fullsized":"2024-03-07T03:09:34.981638+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:39.200015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.984415+0000","last_change":"2024-03-07T03:09:31.492946+0000","last_active":"2024-03-07T03:09:34.984415+0000","last_peered":"2024-03-07T03:09:34.984415+0000","last_clean":"2024-03-07T03:09:34.984415+0000","last_became_active":"2024-03-07T03:09:31.491949+0000","last_became_peered":"2024-03-07T03:09:31.491949+0000","last_unstale":"2024-03-07T03:09:34.984415+0000","last_undegraded":"2024-03-07T03:09:34.984415+0000","last_fullsized":"2024-03-07T03:09:34.984415+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:42:06.966194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.981601+0000","last_change":"2024-03-07T03:09:31.340350+0000","last_active":"2024-03-07T03:09:34.981601+0000","last_peered":"2024-03-07T03:09:34.981601+0000","last_clean":"2024-03-07T03:09:34.981601+0000","last_became_active":"2024-03-07T03:09:31.339524+0000","last_became_peered":"2024-03-07T03:09:31.339524+0000","last_unstale":"2024-03-07T03:09:34.981601+0000","last_undegraded":"2024-03-07T03:09:34.981601+0000","last_fullsized":"2024-03-07T03:09:34.981601+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:14:32.989838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.980963+0000","last_change":"2024-03-07T03:09:31.493118+0000","last_active":"2024-03-07T03:09:34.980963+0000","last_peered":"2024-03-07T03:09:34.980963+0000","last_clean":"2024-03-07T03:09:34.980963+0000","last_became_active":"2024-03-07T03:09:31.491952+0000","last_became_peered":"2024-03-07T03:09:31.491952+0000","last_unstale":"2024-03-07T03:09:34.980963+0000","last_undegraded":"2024-03-07T03:09:34.980963+0000","last_fullsized":"2024-03-07T03:09:34.980963+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:52:26.178237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28224,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716928,"statfs":{"total":95995035648,"available":95966134272,"internally_reserved":0,"allocated":1232896,"data_stored":856851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.89500000000000002,"15min":0.88700000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.2849999999999999,"5min":1.327,"15min":1.327},"last":1.252},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.96699999999999997,"15min":0.96499999999999997},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":2.1909999999999998,"5min":2.1909999999999998,"15min":2.1909999999999998},"last":1.071}]},{"osd":1,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.002,"15min":0.98799999999999999},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":2.274,"5min":2.274,"15min":2.274},"last":1.2749999999999999},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.95699999999999996,"15min":0.97299999999999998},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.1619999999999999,"5min":1.536,"15min":1.536},"last":0.81499999999999995}]},{"osd":2,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.079,"15min":1.105},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.27,"5min":4.0270000000000001,"15min":4.0270000000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.032,"15min":1.038},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.93400000000000005}]},{"osd":3,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":2.0710000000000002,"5min":2.0710000000000002,"15min":2.0710000000000002},"last":1.2270000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.996}]},{"osd":4,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":0.98299999999999998,"15min":0.96699999999999997},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.831,"5min":1.831,"15min":1.831},"last":1.1419999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.98199999999999998,"15min":0.97799999999999998},"min":{"1min":0.53500000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.8080000000000001,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":1.1619999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0029999999999999,"15min":0.99299999999999999},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.8899999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.1870000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0029999999999999,"15min":0.98799999999999999},"min":{"1min":0.63600000000000001,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.1419999999999999,"5min":2.1419999999999999,"15min":2.1419999999999999},"last":1.204}]},{"osd":6,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.0860000000000001,"15min":1.083},"min":{"1min":0.76800000000000002,"5min":0.503,"15min":0.503},"max":{"1min":2.0459999999999998,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0429999999999999,"15min":1.0329999999999999},"min":{"1min":0.68100000000000005,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":1.3}]}]},{"osd":6,"up_from":47,"seq":201863462960,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1228800,"data_stored":856578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.84799999999999998,"15min":0.83199999999999996},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":1.1519999999999999},{"interface":"front","average":{"1min":0.878,"5min":0.91900000000000004,"15min":0.93300000000000005},"min":{"1min":0.60599999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.099}]},{"osd":1,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.90500000000000003,"15min":0.91900000000000004},"min":{"1min":0.54900000000000004,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.206,"5min":1.258,"15min":1.258},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.92800000000000005,"15min":0.92600000000000005},"min":{"1min":0.65700000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":1.25}]},{"osd":2,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.94799999999999995,"15min":0.93700000000000006},"min":{"1min":0.53100000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.93600000000000005},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.94699999999999995,"15min":0.95499999999999996},"min":{"1min":0.67700000000000005,"5min":0.37,"15min":0.37},"max":{"1min":1.7529999999999999,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":0.79600000000000004}]},{"osd":3,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.94899999999999995,"15min":0.95099999999999996},"min":{"1min":0.52000000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.1899999999999999,"5min":1.3,"15min":1.3},"last":0.88200000000000001},{"interface":"front","average":{"1min":1.02,"5min":0.997,"15min":0.99199999999999999},"min":{"1min":0.60399999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":1.3120000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.85899999999999999,"15min":0.85499999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.2949999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.90700000000000003,"15min":0.90200000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.2749999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.93700000000000006,"15min":0.92500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.8759999999999999,"5min":1.8759999999999999,"15min":1.8759999999999999},"last":1.22},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.661,"5min":1.661,"15min":1.661},"last":1.1950000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.032,"15min":1.018},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.994,"5min":1.994,"15min":1.994},"last":1.6040000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.079,"15min":1.069},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.8220000000000001,"5min":1.8220000000000001,"15min":1.8220000000000001},"last":1.3540000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574945,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.70999999999999996,"15min":0.63800000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.246},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.4199999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.80900000000000005,"15min":0.65400000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.32100000000000001},"max":{"1min":1.3049999999999999,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":1.032}]},{"osd":2,"last update":"Thu Mar 7 03:09:48 2024","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.76400000000000001,"15min":0.65800000000000003},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.28599999999999998},"max":{"1min":1.2310000000000001,"5min":1.2310000000000001,"15min":1.2310000000000001},"last":0.85099999999999998},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.75700000000000001,"15min":0.64100000000000001},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.30499999999999999},"max":{"1min":1.0189999999999999,"5min":1.075,"15min":1.075},"last":0.76200000000000001}]},{"osd":3,"last update":"Thu Mar 7 03:10:48 2024","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.79000000000000004},"min":{"1min":0.438,"5min":0.372,"15min":0.372},"max":{"1min":1.2350000000000001,"5min":1.518,"15min":1.518},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.871,"5min":0.91100000000000003,"15min":0.86699999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3,"5min":1.456,"15min":1.456},"last":0.95399999999999996}]},{"osd":4,"last update":"Thu Mar 7 03:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.90600000000000003},"min":{"1min":0.56899999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.0289999999999999,"15min":0.98699999999999999},"min":{"1min":0.69999999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4630000000000001,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":0.90400000000000003}]},{"osd":5,"last update":"Thu Mar 7 03:10:21 2024","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0289999999999999,"15min":0.98299999999999998},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.446,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":1.135},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.016,"15min":1.008},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.476,"5min":1.476,"15min":1.476},"last":1.341}]},{"osd":6,"last update":"Thu Mar 7 03:10:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.085,"15min":1.083},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.403,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.165},{"interface":"front","average":{"1min":1.155,"5min":1.1220000000000001,"15min":1.109},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":1.294}]},{"osd":7,"last update":"Thu Mar 7 03:10:00 2024","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.085,"15min":1.079},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.556,"5min":1.556,"15min":1.556},"last":1.262},{"interface":"front","average":{"1min":1.145,"5min":1.1850000000000001,"15min":1.1910000000000001},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.486,"5min":1.9530000000000001,"15min":1.9530000000000001},"last":1.216}]}]},{"osd":0,"up_from":8,"seq":34359738475,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28228,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716924,"statfs":{"total":95995035648,"available":95966130176,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.68400000000000005,"15min":0.58199999999999996},"min":{"1min":0.437,"5min":0.40000000000000002,"15min":0.246},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.76100000000000001,"15min":0.626},"min":{"1min":0.314,"5min":0.314,"15min":0.23200000000000001},"max":{"1min":1.264,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.97099999999999997}]},{"osd":2,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.82199999999999995,"15min":0.70399999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.29099999999999998},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.76600000000000001,"15min":0.69399999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.34899999999999998},"max":{"1min":1.0920000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.754}]},{"osd":3,"last update":"Thu Mar 7 03:10:46 2024","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.89100000000000001,"15min":0.80500000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.788,"5min":1.788,"15min":1.788},"last":1.159},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.84499999999999997,"15min":0.80400000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.5820000000000001,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":1.1839999999999999}]},{"osd":4,"last update":"Thu Mar 7 03:10:34 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.91000000000000003,"15min":0.86799999999999999},"min":{"1min":0.70399999999999996,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.079},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0369999999999999,"15min":0.97299999999999998},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3939999999999999,"5min":1.46,"15min":1.46},"last":1.0609999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:10:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0640000000000001,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.649,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.256},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.04,"15min":1.0009999999999999},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.4730000000000001,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.0089999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.1279999999999999,"15min":1.1339999999999999},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.587,"5min":1.609,"15min":1.609},"last":1.3080000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.107},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":1.131}]},{"osd":7,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.089,"15min":1.095},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.224},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.2410000000000001,"15min":1.2490000000000001},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.73,"5min":1.73,"15min":1.73},"last":1.359}]}]},{"osd":2,"up_from":18,"seq":77309411415,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.70799999999999996,"15min":0.627},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.2929999999999999},"last":0.625},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.77700000000000002,"15min":0.63900000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.71099999999999997}]},{"osd":1,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.747,"15min":0.66700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.33200000000000002},"max":{"1min":1.0309999999999999,"5min":1.218,"15min":1.218},"last":1.0069999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.82099999999999995,"15min":0.70799999999999996},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.312},"max":{"1min":1.2010000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.57399999999999995}]},{"osd":3,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.80700000000000005,"15min":0.79500000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.0620000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.81599999999999995},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.877,"15min":0.83899999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.1699999999999999,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":0.78800000000000003}]},{"osd":4,"last update":"Thu Mar 7 03:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.0009999999999999,"15min":0.98299999999999998},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":0.98199999999999998},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.91700000000000004,"15min":0.92000000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.1930000000000001,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.85199999999999998}]},{"osd":5,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.0289999999999999,"15min":1.0309999999999999},"min":{"1min":0.77300000000000002,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.2330000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.95099999999999996,"15min":0.96999999999999997},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.2450000000000001,"5min":1.25,"15min":1.25},"last":0.94799999999999995}]},{"osd":6,"last update":"Thu Mar 7 03:10:09 2024","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.091,"15min":1.119},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.333,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.129,"15min":1.1459999999999999},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.403,"5min":1.5700000000000001,"15min":1.5700000000000001},"last":1.038}]},{"osd":7,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.04,"15min":1.046},"min":{"1min":0.73999999999999999,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.214,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.1359999999999999,"15min":1.143},"min":{"1min":0.68200000000000005,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.5669999999999999,"5min":1.611,"15min":1.611},"last":1.1910000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149774,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.68100000000000005,"15min":0.67500000000000004},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.6419999999999999},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.76100000000000001,"15min":0.72699999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.1759999999999999,"5min":1.321,"15min":1.321},"last":1.075}]},{"osd":1,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.78500000000000003,"15min":0.80000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.1160000000000001,"5min":1.2709999999999999,"15min":1.4550000000000001},"last":0.91100000000000003},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.755,"15min":0.73999999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.94799999999999995,"5min":1.0860000000000001,"15min":1.095},"last":0.97599999999999998}]},{"osd":2,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.83799999999999997,"15min":0.81599999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.046,"5min":1.198,"15min":1.5109999999999999},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.98099999999999998,"5min":1.1080000000000001,"15min":1.1080000000000001},"last":0.94299999999999995}]},{"osd":4,"last update":"Thu Mar 7 03:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.98099999999999998,"15min":0.93300000000000005},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":1.157},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.0109999999999999,"15min":0.97199999999999998},"min":{"1min":0.745,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":1.272}]},{"osd":5,"last update":"Thu Mar 7 03:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.94699999999999995,"15min":0.95999999999999996},"min":{"1min":0.67700000000000005,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.0880000000000001,"5min":1.268,"15min":1.268},"last":1.115},{"interface":"front","average":{"1min":1.05,"5min":1.071,"15min":1.0669999999999999},"min":{"1min":0.66200000000000003,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.48,"5min":1.498,"15min":1.498},"last":1.1970000000000001}]},{"osd":6,"last update":"Thu Mar 7 03:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.089,"15min":1.111},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":1.135},{"interface":"front","average":{"1min":1.014,"5min":1.032,"15min":1.048},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.0209999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:09:56 2024","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.206,"15min":1.214},"min":{"1min":0.78200000000000003,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.2090000000000001,"15min":1.2290000000000001},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.2230000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986243,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.83099999999999996,"15min":0.78800000000000003},"min":{"1min":0.65600000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.222,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.86899999999999999,"15min":0.83199999999999996},"min":{"1min":0.59599999999999997,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":1.252}]},{"osd":1,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.89500000000000002,"15min":0.85899999999999999},"min":{"1min":0.63300000000000001,"5min":0.497,"15min":0.497},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.97299999999999998},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.89900000000000002,"15min":0.84699999999999998},"min":{"1min":0.69299999999999995,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.214,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.0089999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.90300000000000002,"15min":0.89100000000000001},"min":{"1min":0.67000000000000004,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.26,"5min":1.26,"15min":1.26},"last":0.82199999999999995},{"interface":"front","average":{"1min":1.02,"5min":0.93999999999999995,"15min":0.875},"min":{"1min":0.65200000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3089999999999999,"5min":1.323,"15min":1.323},"last":1.0780000000000001}]},{"osd":3,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.91100000000000003,"15min":0.85199999999999998},"min":{"1min":0.72699999999999998,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.327,"5min":1.327,"15min":1.327},"last":1.161},{"interface":"front","average":{"1min":1.091,"5min":0.96299999999999997,"15min":0.91800000000000004},"min":{"1min":0.754,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.367,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":0.88}]},{"osd":5,"last update":"Thu Mar 7 03:10:21 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.82299999999999995,"15min":0.75},"min":{"1min":0.68500000000000005,"5min":0.371,"15min":0.371},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":0.77500000000000002},{"interface":"front","average":{"1min":1.012,"5min":0.85599999999999998,"15min":0.80600000000000005},"min":{"1min":0.63400000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.0449999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:04 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.94799999999999995,"15min":0.90000000000000002},"min":{"1min":0.49199999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.103},{"interface":"front","average":{"1min":1.095,"5min":0.96299999999999997,"15min":0.90100000000000002},"min":{"1min":0.51900000000000002,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.754,"5min":1.754,"15min":1.754},"last":1.1359999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:09:55 2024","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.022,"15min":1.0069999999999999},"min":{"1min":0.47199999999999998,"5min":0.436,"15min":0.436},"max":{"1min":1.4630000000000001,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.123,"15min":1.111},"min":{"1min":0.623,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.681,"5min":1.681,"15min":1.681},"last":1.208}]}]},{"osd":5,"up_from":39,"seq":167503724602,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.90100000000000002,"15min":0.89900000000000002},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.1910000000000001,"5min":1.597,"15min":1.597},"last":1.0009999999999999},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.89100000000000001,"15min":0.85599999999999998},"min":{"1min":0.67600000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.284,"5min":1.3,"15min":1.3},"last":0.98199999999999998}]},{"osd":1,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.92700000000000005,"15min":0.89700000000000002},"min":{"1min":0.63,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.2949999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.1859999999999999},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.90700000000000003,"15min":0.89100000000000001},"min":{"1min":0.73299999999999998,"5min":0.501,"15min":0.501},"max":{"1min":1.252,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.028}]},{"osd":2,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.89900000000000002,"15min":0.875},"min":{"1min":0.5,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.999,"5min":0.93400000000000005,"15min":0.91600000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.91200000000000003}]},{"osd":3,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.93500000000000005,"15min":0.92100000000000004},"min":{"1min":0.60499999999999998,"5min":0.497,"15min":0.497},"max":{"1min":1.528,"5min":1.528,"15min":1.528},"last":0.95599999999999996},{"interface":"front","average":{"1min":1.0569999999999999,"5min":0.999,"15min":0.98499999999999999},"min":{"1min":0.61499999999999999,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":1.1040000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.82499999999999996,"15min":0.755},"min":{"1min":0.42699999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.47,"5min":1.571,"15min":1.571},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.05,"5min":0.87,"15min":0.81799999999999995},"min":{"1min":0.44900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.0469999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:10 2024","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":0.91000000000000003,"15min":0.85199999999999998},"min":{"1min":0.69899999999999995,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.911,"5min":1.911,"15min":1.911},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.1259999999999999,"5min":0.93400000000000005,"15min":0.874},"min":{"1min":0.40899999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.679,"5min":1.679,"15min":1.679},"last":0.81799999999999995}]},{"osd":7,"last update":"Thu Mar 7 03:09:59 2024","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.0489999999999999,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.7,"5min":1.7,"15min":1.7},"last":0.752},{"interface":"front","average":{"1min":1.214,"5min":1.1020000000000001,"15min":1.0840000000000001},"min":{"1min":0.96499999999999997,"5min":0.746,"15min":0.746},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":1.0700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:10:52.320 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:10:52.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:52 smithi148 bash[21362]: cluster 2024-03-07T03:10:51.001943+0000 mgr.y (mgr.24287) 95 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:52.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:52 smithi148 bash[21362]: audit 2024-03-07T03:10:51.672215+0000 mgr.y (mgr.24287) 96 : audit 0 from='client.24458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:52.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:52 smithi148 bash[31149]: cluster 2024-03-07T03:10:51.001943+0000 mgr.y (mgr.24287) 95 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:52.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:52 smithi148 bash[31149]: audit 2024-03-07T03:10:51.672215+0000 mgr.y (mgr.24287) 96 : audit 0 from='client.24458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:52.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:52 smithi195 bash[23221]: cluster 2024-03-07T03:10:51.001943+0000 mgr.y (mgr.24287) 95 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:52.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:52 smithi195 bash[23221]: audit 2024-03-07T03:10:51.672215+0000 mgr.y (mgr.24287) 96 : audit 0 from='client.24458 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:10:54.634 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:54 smithi148 bash[21362]: cluster 2024-03-07T03:10:53.002539+0000 mgr.y (mgr.24287) 97 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:54.634 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:54 smithi148 bash[31149]: cluster 2024-03-07T03:10:53.002539+0000 mgr.y (mgr.24287) 97 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:54.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:54 smithi195 bash[23221]: cluster 2024-03-07T03:10:53.002539+0000 mgr.y (mgr.24287) 97 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:55.042 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:10:54 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:10:54] "GET /metrics HTTP/1.1" 200 28552 "" "Prometheus/2.43.0" 2024-03-07T03:10:56.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:56 smithi148 bash[21362]: cluster 2024-03-07T03:10:55.003126+0000 mgr.y (mgr.24287) 98 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:56.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:56 smithi148 bash[31149]: cluster 2024-03-07T03:10:55.003126+0000 mgr.y (mgr.24287) 98 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:56.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:56 smithi195 bash[23221]: cluster 2024-03-07T03:10:55.003126+0000 mgr.y (mgr.24287) 98 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:57.039 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:10:58.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:10:58 smithi148 bash[21362]: cluster 2024-03-07T03:10:57.003714+0000 mgr.y (mgr.24287) 99 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:58.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:10:58 smithi148 bash[31149]: cluster 2024-03-07T03:10:57.003714+0000 mgr.y (mgr.24287) 99 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:58.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:10:58 smithi195 bash[23221]: cluster 2024-03-07T03:10:57.003714+0000 mgr.y (mgr.24287) 99 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:10:59.617 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:10:59.619 INFO:teuthology.orchestra.run.smithi148.stderr:dumped all 2024-03-07T03:11:00.251 INFO:teuthology.orchestra.run.smithi148.stdout:{"pg_ready":true,"pg_map":{"version":72,"stamp":"2024-03-07T03:10:59.003973+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":222904,"kb_used_data":6744,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749738312,"statfs":{"total":767960285184,"available":767732031488,"internally_reserved":0,"allocated":6905856,"data_stored":3904077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003165"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.989452+0000","last_change":"2024-03-07T03:09:31.339012+0000","last_active":"2024-03-07T03:09:34.989452+0000","last_peered":"2024-03-07T03:09:34.989452+0000","last_clean":"2024-03-07T03:09:34.989452+0000","last_became_active":"2024-03-07T03:09:31.338487+0000","last_became_peered":"2024-03-07T03:09:31.338487+0000","last_unstale":"2024-03-07T03:09:34.989452+0000","last_undegraded":"2024-03-07T03:09:34.989452+0000","last_fullsized":"2024-03-07T03:09:34.989452+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:48:57.153468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.981001+0000","last_change":"2024-03-07T03:09:31.492237+0000","last_active":"2024-03-07T03:09:34.981001+0000","last_peered":"2024-03-07T03:09:34.981001+0000","last_clean":"2024-03-07T03:09:34.981001+0000","last_became_active":"2024-03-07T03:09:31.490659+0000","last_became_peered":"2024-03-07T03:09:31.490659+0000","last_unstale":"2024-03-07T03:09:34.981001+0000","last_undegraded":"2024-03-07T03:09:34.981001+0000","last_fullsized":"2024-03-07T03:09:34.981001+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:58:14.548445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.983783+0000","last_change":"2024-03-07T03:09:31.490148+0000","last_active":"2024-03-07T03:09:34.983783+0000","last_peered":"2024-03-07T03:09:34.983783+0000","last_clean":"2024-03-07T03:09:34.983783+0000","last_became_active":"2024-03-07T03:09:31.489708+0000","last_became_peered":"2024-03-07T03:09:31.489708+0000","last_unstale":"2024-03-07T03:09:34.983783+0000","last_undegraded":"2024-03-07T03:09:34.983783+0000","last_fullsized":"2024-03-07T03:09:34.983783+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:21:06.820876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.983692+0000","last_change":"2024-03-07T03:09:31.490070+0000","last_active":"2024-03-07T03:09:34.983692+0000","last_peered":"2024-03-07T03:09:34.983692+0000","last_clean":"2024-03-07T03:09:34.983692+0000","last_became_active":"2024-03-07T03:09:31.489632+0000","last_became_peered":"2024-03-07T03:09:31.489632+0000","last_unstale":"2024-03-07T03:09:34.983692+0000","last_undegraded":"2024-03-07T03:09:34.983692+0000","last_fullsized":"2024-03-07T03:09:34.983692+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:08:00.120410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":260,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-03-07T03:09:39.563927+0000","last_change":"2024-03-07T03:07:57.209434+0000","last_active":"2024-03-07T03:09:39.563927+0000","last_peered":"2024-03-07T03:09:39.563927+0000","last_clean":"2024-03-07T03:09:39.563927+0000","last_became_active":"2024-03-07T03:07:57.208461+0000","last_became_peered":"2024-03-07T03:07:57.208461+0000","last_unstale":"2024-03-07T03:09:39.563927+0000","last_undegraded":"2024-03-07T03:09:39.563927+0000","last_fullsized":"2024-03-07T03:09:39.563927+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_clean_scrub_stamp":"2024-03-07T03:03:44.169894+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:55:26.915133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.981638+0000","last_change":"2024-03-07T03:09:31.338124+0000","last_active":"2024-03-07T03:09:34.981638+0000","last_peered":"2024-03-07T03:09:34.981638+0000","last_clean":"2024-03-07T03:09:34.981638+0000","last_became_active":"2024-03-07T03:09:31.337531+0000","last_became_peered":"2024-03-07T03:09:31.337531+0000","last_unstale":"2024-03-07T03:09:34.981638+0000","last_undegraded":"2024-03-07T03:09:34.981638+0000","last_fullsized":"2024-03-07T03:09:34.981638+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:39.200015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.984415+0000","last_change":"2024-03-07T03:09:31.492946+0000","last_active":"2024-03-07T03:09:34.984415+0000","last_peered":"2024-03-07T03:09:34.984415+0000","last_clean":"2024-03-07T03:09:34.984415+0000","last_became_active":"2024-03-07T03:09:31.491949+0000","last_became_peered":"2024-03-07T03:09:31.491949+0000","last_unstale":"2024-03-07T03:09:34.984415+0000","last_undegraded":"2024-03-07T03:09:34.984415+0000","last_fullsized":"2024-03-07T03:09:34.984415+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:42:06.966194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.981601+0000","last_change":"2024-03-07T03:09:31.340350+0000","last_active":"2024-03-07T03:09:34.981601+0000","last_peered":"2024-03-07T03:09:34.981601+0000","last_clean":"2024-03-07T03:09:34.981601+0000","last_became_active":"2024-03-07T03:09:31.339524+0000","last_became_peered":"2024-03-07T03:09:31.339524+0000","last_unstale":"2024-03-07T03:09:34.981601+0000","last_undegraded":"2024-03-07T03:09:34.981601+0000","last_fullsized":"2024-03-07T03:09:34.981601+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:14:32.989838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-07T03:09:34.980963+0000","last_change":"2024-03-07T03:09:31.493118+0000","last_active":"2024-03-07T03:09:34.980963+0000","last_peered":"2024-03-07T03:09:34.980963+0000","last_clean":"2024-03-07T03:09:34.980963+0000","last_became_active":"2024-03-07T03:09:31.491952+0000","last_became_peered":"2024-03-07T03:09:31.491952+0000","last_unstale":"2024-03-07T03:09:34.980963+0000","last_undegraded":"2024-03-07T03:09:34.980963+0000","last_fullsized":"2024-03-07T03:09:34.980963+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:52:26.178237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28224,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716928,"statfs":{"total":95995035648,"available":95966134272,"internally_reserved":0,"allocated":1232896,"data_stored":856851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.89500000000000002,"15min":0.88700000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.2849999999999999,"5min":1.327,"15min":1.327},"last":0.98799999999999999},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.96699999999999997,"15min":0.96499999999999997},"min":{"1min":0.56100000000000005,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":2.1909999999999998,"5min":2.1909999999999998,"15min":2.1909999999999998},"last":0.95599999999999996}]},{"osd":1,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":1.002,"15min":0.98799999999999999},"min":{"1min":0.627,"5min":0.627,"15min":0.627},"max":{"1min":2.274,"5min":2.274,"15min":2.274},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.86499999999999999,"5min":0.95699999999999996,"15min":0.97299999999999998},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.1619999999999999,"5min":1.536,"15min":1.536},"last":1.0589999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.079,"15min":1.105},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.27,"5min":4.0270000000000001,"15min":4.0270000000000001},"last":1.1919999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.032,"15min":1.038},"min":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.6719999999999999,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":0.66600000000000004}]},{"osd":3,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.0840000000000001,"15min":1.0840000000000001},"min":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":2.0710000000000002,"5min":2.0710000000000002,"15min":2.0710000000000002},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"min":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.4890000000000001,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":1.173}]},{"osd":4,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.081,"5min":0.98299999999999998,"15min":0.96699999999999997},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.831,"5min":1.831,"15min":1.831},"last":1.0329999999999999},{"interface":"front","average":{"1min":1.01,"5min":0.98199999999999998,"15min":0.97799999999999998},"min":{"1min":0.53500000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.8080000000000001,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":1.087}]},{"osd":5,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.0029999999999999,"15min":0.99299999999999999},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.8899999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.80800000000000005},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.0029999999999999,"15min":0.98799999999999999},"min":{"1min":0.63600000000000001,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":2.1419999999999999,"5min":2.1419999999999999,"15min":2.1419999999999999},"last":1.125}]},{"osd":6,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.0860000000000001,"15min":1.083},"min":{"1min":0.76800000000000002,"5min":0.503,"15min":0.503},"max":{"1min":2.0459999999999998,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":0.90900000000000003},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0429999999999999,"15min":1.0329999999999999},"min":{"1min":0.68100000000000005,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":1.724,"5min":1.724,"15min":1.724},"last":1.1439999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462961,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1228800,"data_stored":856578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.84799999999999998,"15min":0.83199999999999996},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":1.409,"5min":1.409,"15min":1.409},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.878,"5min":0.91900000000000004,"15min":0.93300000000000005},"min":{"1min":0.60599999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.363,"5min":1.363,"15min":1.363},"last":1.0309999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.90500000000000003,"15min":0.91900000000000004},"min":{"1min":0.54900000000000004,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.206,"5min":1.258,"15min":1.258},"last":0.84899999999999998},{"interface":"front","average":{"1min":1.0229999999999999,"5min":0.92800000000000005,"15min":0.92600000000000005},"min":{"1min":0.65700000000000003,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.339,"5min":1.339,"15min":1.339},"last":1.1899999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":0.94799999999999995,"15min":0.93700000000000006},"min":{"1min":0.53100000000000003,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.4550000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.0800000000000001},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.94699999999999995,"15min":0.95499999999999996},"min":{"1min":0.67700000000000005,"5min":0.37,"15min":0.37},"max":{"1min":1.7529999999999999,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":0.79500000000000004}]},{"osd":3,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.94899999999999995,"15min":0.95099999999999996},"min":{"1min":0.52000000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.1899999999999999,"5min":1.3,"15min":1.3},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.02,"5min":0.997,"15min":0.99199999999999999},"min":{"1min":0.60399999999999998,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":0.93400000000000005}]},{"osd":4,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":0.85899999999999999,"15min":0.85499999999999998},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2909999999999999,"5min":1.2909999999999999,"15min":1.2909999999999999},"last":1.228},{"interface":"front","average":{"1min":0.93100000000000005,"5min":0.90700000000000003,"15min":0.90200000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.4810000000000001,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.98799999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:10:07 2024","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.93700000000000006,"15min":0.92500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.8759999999999999,"5min":1.8759999999999999,"15min":1.8759999999999999},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.0269999999999999,"5min":0.99299999999999999,"15min":0.99299999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":1.661,"5min":1.661,"15min":1.661},"last":1.1200000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.032,"15min":1.018},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":1.994,"5min":1.994,"15min":1.994},"last":1.2769999999999999},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.079,"15min":1.069},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":1.8220000000000001,"5min":1.8220000000000001,"15min":1.8220000000000001},"last":1.306}]}]},{"osd":1,"up_from":13,"seq":55834574946,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:03 2024","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.70999999999999996,"15min":0.63800000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.246},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.4199999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.80900000000000005,"15min":0.65400000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.32100000000000001},"max":{"1min":1.3049999999999999,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":1.0609999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:10:51 2024","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":0.755,"15min":0.66600000000000004},"min":{"1min":0.41599999999999998,"5min":0.33800000000000002,"15min":0.28599999999999998},"max":{"1min":1.089,"5min":1.2310000000000001,"15min":1.2310000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.76400000000000001,"15min":0.65100000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":1.3520000000000001,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.80900000000000005}]},{"osd":3,"last update":"Thu Mar 7 03:10:48 2024","interfaces":[{"interface":"back","average":{"1min":0.76800000000000002,"5min":0.80000000000000004,"15min":0.79000000000000004},"min":{"1min":0.438,"5min":0.372,"15min":0.372},"max":{"1min":1.2350000000000001,"5min":1.518,"15min":1.518},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.871,"5min":0.91100000000000003,"15min":0.86699999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.3,"5min":1.456,"15min":1.456},"last":0.92900000000000005}]},{"osd":4,"last update":"Thu Mar 7 03:10:31 2024","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.90600000000000003},"min":{"1min":0.56899999999999995,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":1.3120000000000001},{"interface":"front","average":{"1min":1.026,"5min":1.0289999999999999,"15min":0.98699999999999999},"min":{"1min":0.69999999999999996,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.4630000000000001,"5min":1.7729999999999999,"15min":1.7729999999999999},"last":1.216}]},{"osd":5,"last update":"Thu Mar 7 03:10:21 2024","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.0289999999999999,"15min":0.98299999999999998},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.446,"5min":1.6579999999999999,"15min":1.6579999999999999},"last":1.0209999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.016,"15min":1.008},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.476,"5min":1.476,"15min":1.476},"last":1.246}]},{"osd":6,"last update":"Thu Mar 7 03:10:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0609999999999999,"5min":1.085,"15min":1.083},"min":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.403,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.3380000000000001},{"interface":"front","average":{"1min":1.155,"5min":1.1220000000000001,"15min":1.109},"min":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":1.1819999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:10:00 2024","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":1.085,"15min":1.079},"min":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.556,"5min":1.556,"15min":1.556},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.145,"5min":1.1850000000000001,"15min":1.1910000000000001},"min":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":1.486,"5min":1.9530000000000001,"15min":1.9530000000000001},"last":1.2729999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738476,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28228,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716924,"statfs":{"total":95995035648,"available":95966130176,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":0.68400000000000005,"15min":0.58199999999999996},"min":{"1min":0.437,"5min":0.40000000000000002,"15min":0.246},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.76100000000000001,"15min":0.626},"min":{"1min":0.314,"5min":0.314,"15min":0.23200000000000001},"max":{"1min":1.264,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.70199999999999996}]},{"osd":2,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.82199999999999995,"15min":0.70399999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.29099999999999998},"max":{"1min":1.4239999999999999,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":0.94799999999999995},{"interface":"front","average":{"1min":0.75800000000000001,"5min":0.76600000000000001,"15min":0.69399999999999995},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.34899999999999998},"max":{"1min":1.0920000000000001,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.84299999999999997}]},{"osd":3,"last update":"Thu Mar 7 03:10:46 2024","interfaces":[{"interface":"back","average":{"1min":0.82499999999999996,"5min":0.89100000000000001,"15min":0.80500000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":1.788,"5min":1.788,"15min":1.788},"last":0.86099999999999999},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.84499999999999997,"15min":0.80400000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.5820000000000001,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":0.98999999999999999}]},{"osd":4,"last update":"Thu Mar 7 03:10:34 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.91000000000000003,"15min":0.86799999999999999},"min":{"1min":0.70399999999999996,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.5409999999999999,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":1.2190000000000001},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0369999999999999,"15min":0.97299999999999998},"min":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.3939999999999999,"5min":1.46,"15min":1.46},"last":1.115}]},{"osd":5,"last update":"Thu Mar 7 03:10:23 2024","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.0640000000000001,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"max":{"1min":1.649,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.048},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.04,"15min":1.0009999999999999},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.4730000000000001,"5min":1.4730000000000001,"15min":1.4730000000000001},"last":1.343}]},{"osd":6,"last update":"Thu Mar 7 03:10:11 2024","interfaces":[{"interface":"back","average":{"1min":1.0640000000000001,"5min":1.1279999999999999,"15min":1.1339999999999999},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.587,"5min":1.609,"15min":1.609},"last":1.1830000000000001},{"interface":"front","average":{"1min":1.1100000000000001,"5min":1.1100000000000001,"15min":1.107},"min":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":1.2889999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:10:01 2024","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.089,"15min":1.095},"min":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"max":{"1min":1.5920000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.1359999999999999},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.2410000000000001,"15min":1.2490000000000001},"min":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"max":{"1min":1.73,"5min":1.73,"15min":1.73},"last":1.2689999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411416,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.70799999999999996,"15min":0.627},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.1459999999999999,"5min":1.1459999999999999,"15min":1.2929999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.77700000000000002,"15min":0.63900000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.194,"5min":1.194,"15min":1.194},"last":0.78600000000000003}]},{"osd":1,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.747,"15min":0.66700000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.33200000000000002},"max":{"1min":1.0309999999999999,"5min":1.218,"15min":1.218},"last":0.871},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.82099999999999995,"15min":0.70799999999999996},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.312},"max":{"1min":1.2010000000000001,"5min":1.2430000000000001,"15min":1.2430000000000001},"last":0.95399999999999996}]},{"osd":3,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.80700000000000005,"15min":0.79500000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.0620000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":1.157},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.877,"15min":0.83899999999999997},"min":{"1min":0.438,"5min":0.438,"15min":0.438},"max":{"1min":1.1699999999999999,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":1.2}]},{"osd":4,"last update":"Thu Mar 7 03:10:32 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":1.0009999999999999,"15min":0.98299999999999998},"min":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.3440000000000001,"5min":1.3440000000000001,"15min":1.3440000000000001},"last":1.111},{"interface":"front","average":{"1min":0.90900000000000003,"5min":0.91700000000000004,"15min":0.92000000000000004},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.1930000000000001,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":1.01}]},{"osd":5,"last update":"Thu Mar 7 03:10:13 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":1.0289999999999999,"15min":1.0309999999999999},"min":{"1min":0.77300000000000002,"5min":0.70599999999999996,"15min":0.70599999999999996},"max":{"1min":1.2330000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.2729999999999999},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.95099999999999996,"15min":0.96999999999999997},"min":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.2450000000000001,"5min":1.25,"15min":1.25},"last":1.23}]},{"osd":6,"last update":"Thu Mar 7 03:10:09 2024","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.091,"15min":1.119},"min":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.333,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":1.343},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.129,"15min":1.1459999999999999},"min":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.403,"5min":1.5700000000000001,"15min":1.5700000000000001},"last":1.371}]},{"osd":7,"last update":"Thu Mar 7 03:09:58 2024","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":1.04,"15min":1.046},"min":{"1min":0.73999999999999999,"5min":0.70499999999999996,"15min":0.70499999999999996},"max":{"1min":1.214,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":1.3100000000000001},{"interface":"front","average":{"1min":1.093,"5min":1.1359999999999999,"15min":1.143},"min":{"1min":0.68200000000000005,"5min":0.63600000000000001,"15min":0.63600000000000001},"max":{"1min":1.5669999999999999,"5min":1.611,"15min":1.611},"last":1.417}]}]},{"osd":3,"up_from":26,"seq":111669149775,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.68100000000000005,"15min":0.67500000000000004},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.0580000000000001,"5min":1.0580000000000001,"15min":1.6419999999999999},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.76100000000000001,"15min":0.72699999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.1759999999999999,"5min":1.321,"15min":1.321},"last":0.999}]},{"osd":1,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.753,"5min":0.78500000000000003,"15min":0.80000000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.1160000000000001,"5min":1.2709999999999999,"15min":1.4550000000000001},"last":0.94199999999999995},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.755,"15min":0.73999999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.94799999999999995,"5min":1.0860000000000001,"15min":1.095},"last":1.127}]},{"osd":2,"last update":"Thu Mar 7 03:10:44 2024","interfaces":[{"interface":"back","average":{"1min":0.73999999999999999,"5min":0.83799999999999997,"15min":0.81599999999999995},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.046,"5min":1.198,"15min":1.5109999999999999},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.77300000000000002,"15min":0.78500000000000003},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.98099999999999998,"5min":1.1080000000000001,"15min":1.1080000000000001},"last":0.66100000000000003}]},{"osd":4,"last update":"Thu Mar 7 03:10:27 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.98099999999999998,"15min":0.93300000000000005},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.5289999999999999,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.98399999999999999,"5min":1.0109999999999999,"15min":0.97199999999999998},"min":{"1min":0.745,"5min":0.66300000000000003,"15min":0.66300000000000003},"max":{"1min":1.4359999999999999,"5min":1.4359999999999999,"15min":1.4359999999999999},"last":1.169}]},{"osd":5,"last update":"Thu Mar 7 03:10:22 2024","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":0.94699999999999995,"15min":0.95999999999999996},"min":{"1min":0.67700000000000005,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.0880000000000001,"5min":1.268,"15min":1.268},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.05,"5min":1.071,"15min":1.0669999999999999},"min":{"1min":0.66200000000000003,"5min":0.65300000000000002,"15min":0.65300000000000002},"max":{"1min":1.48,"5min":1.498,"15min":1.498},"last":1.1419999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:05 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.089,"15min":1.111},"min":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.502,"5min":1.502,"15min":1.502},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.014,"5min":1.032,"15min":1.048},"min":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.5840000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.034}]},{"osd":7,"last update":"Thu Mar 7 03:09:56 2024","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.206,"15min":1.214},"min":{"1min":0.78200000000000003,"5min":0.71399999999999997,"15min":0.71399999999999997},"max":{"1min":1.6120000000000001,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.0940000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":1.2090000000000001,"15min":1.2290000000000001},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.2110000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986244,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.83099999999999996,"15min":0.78800000000000003},"min":{"1min":0.65600000000000003,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.222,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.94699999999999995,"5min":0.86899999999999999,"15min":0.83199999999999996},"min":{"1min":0.59599999999999997,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.264,"5min":1.264,"15min":1.264},"last":0.78800000000000003}]},{"osd":1,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.89500000000000002,"15min":0.85899999999999999},"min":{"1min":0.63300000000000001,"5min":0.497,"15min":0.497},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.1160000000000001},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.89900000000000002,"15min":0.84699999999999998},"min":{"1min":0.69299999999999995,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.214,"5min":1.3049999999999999,"15min":1.3049999999999999},"last":1.093}]},{"osd":2,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":0.90300000000000002,"15min":0.89100000000000001},"min":{"1min":0.67000000000000004,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":1.26,"5min":1.26,"15min":1.26},"last":1.2130000000000001},{"interface":"front","average":{"1min":1.02,"5min":0.93999999999999995,"15min":0.875},"min":{"1min":0.65200000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.3089999999999999,"5min":1.323,"15min":1.323},"last":0.95899999999999996}]},{"osd":3,"last update":"Thu Mar 7 03:10:25 2024","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":0.91100000000000003,"15min":0.85199999999999998},"min":{"1min":0.72699999999999998,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.327,"5min":1.327,"15min":1.327},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.091,"5min":0.96299999999999997,"15min":0.91800000000000004},"min":{"1min":0.754,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.367,"5min":1.3720000000000001,"15min":1.3720000000000001},"last":1.238}]},{"osd":5,"last update":"Thu Mar 7 03:10:21 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.82299999999999995,"15min":0.75},"min":{"1min":0.68500000000000005,"5min":0.371,"15min":0.371},"max":{"1min":1.2949999999999999,"5min":1.2949999999999999,"15min":1.2949999999999999},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.012,"5min":0.85599999999999998,"15min":0.80600000000000005},"min":{"1min":0.63400000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.1519999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:10:04 2024","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":0.94799999999999995,"15min":0.90000000000000002},"min":{"1min":0.49199999999999999,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.1850000000000001},{"interface":"front","average":{"1min":1.095,"5min":0.96299999999999997,"15min":0.90100000000000002},"min":{"1min":0.51900000000000002,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.754,"5min":1.754,"15min":1.754},"last":1.046}]},{"osd":7,"last update":"Thu Mar 7 03:09:55 2024","interfaces":[{"interface":"back","average":{"1min":1.1100000000000001,"5min":1.022,"15min":1.0069999999999999},"min":{"1min":0.47199999999999998,"5min":0.436,"15min":0.436},"max":{"1min":1.4630000000000001,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":1.26},{"interface":"front","average":{"1min":1.1970000000000001,"5min":1.123,"15min":1.111},"min":{"1min":0.623,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":1.681,"5min":1.681,"15min":1.681},"last":1.284}]}]},{"osd":5,"up_from":39,"seq":167503724603,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":0.90100000000000002,"15min":0.89900000000000002},"min":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.1910000000000001,"5min":1.597,"15min":1.597},"last":0.995},{"interface":"front","average":{"1min":0.92400000000000004,"5min":0.89100000000000001,"15min":0.85599999999999998},"min":{"1min":0.67600000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.284,"5min":1.3,"15min":1.3},"last":0.92000000000000004}]},{"osd":1,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.98499999999999999,"5min":0.92700000000000005,"15min":0.89700000000000002},"min":{"1min":0.63,"5min":0.51700000000000002,"15min":0.51700000000000002},"max":{"1min":1.2949999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.2490000000000001},{"interface":"front","average":{"1min":0.94499999999999995,"5min":0.90700000000000003,"15min":0.89100000000000001},"min":{"1min":0.73299999999999998,"5min":0.501,"15min":0.501},"max":{"1min":1.252,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":0.79600000000000004}]},{"osd":2,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.89900000000000002,"15min":0.875},"min":{"1min":0.5,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.407,"5min":1.407,"15min":1.407},"last":1.1180000000000001},{"interface":"front","average":{"1min":0.999,"5min":0.93400000000000005,"15min":0.91600000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":1.0960000000000001}]},{"osd":3,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.93500000000000005,"15min":0.92100000000000004},"min":{"1min":0.60499999999999998,"5min":0.497,"15min":0.497},"max":{"1min":1.528,"5min":1.528,"15min":1.528},"last":1.042},{"interface":"front","average":{"1min":1.0569999999999999,"5min":0.999,"15min":0.98499999999999999},"min":{"1min":0.61499999999999999,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":1.296}]},{"osd":4,"last update":"Thu Mar 7 03:10:18 2024","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":0.82499999999999996,"15min":0.755},"min":{"1min":0.42699999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.47,"5min":1.571,"15min":1.571},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.05,"5min":0.87,"15min":0.81799999999999995},"min":{"1min":0.44900000000000001,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":0.84299999999999997}]},{"osd":6,"last update":"Thu Mar 7 03:10:10 2024","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":0.91000000000000003,"15min":0.85199999999999998},"min":{"1min":0.69899999999999995,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.911,"5min":1.911,"15min":1.911},"last":1.071},{"interface":"front","average":{"1min":1.1259999999999999,"5min":0.93400000000000005,"15min":0.874},"min":{"1min":0.40899999999999997,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.679,"5min":1.679,"15min":1.679},"last":1.2050000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:09:59 2024","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":1.0489999999999999,"15min":1.0309999999999999},"min":{"1min":0.67000000000000004,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.7,"5min":1.7,"15min":1.7},"last":1.169},{"interface":"front","average":{"1min":1.214,"5min":1.1020000000000001,"15min":1.0840000000000001},"min":{"1min":0.96499999999999997,"5min":0.746,"15min":0.746},"max":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":1.149}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:11:00.253 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-07T03:11:00.253 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-07T03:11:00.253 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-07T03:11:00.254 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:11:00.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:00 smithi148 bash[21362]: cluster 2024-03-07T03:10:59.004179+0000 mgr.y (mgr.24287) 100 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:00.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:00 smithi148 bash[21362]: audit 2024-03-07T03:10:59.621927+0000 mgr.y (mgr.24287) 101 : audit 0 from='client.14553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:11:00.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:00 smithi148 bash[31149]: cluster 2024-03-07T03:10:59.004179+0000 mgr.y (mgr.24287) 100 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:00.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:00 smithi148 bash[31149]: audit 2024-03-07T03:10:59.621927+0000 mgr.y (mgr.24287) 101 : audit 0 from='client.14553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:11:00.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:00 smithi195 bash[23221]: cluster 2024-03-07T03:10:59.004179+0000 mgr.y (mgr.24287) 100 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:00.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:00 smithi195 bash[23221]: audit 2024-03-07T03:10:59.621927+0000 mgr.y (mgr.24287) 101 : audit 0 from='client.14553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:11:02.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:02 smithi148 bash[21362]: cluster 2024-03-07T03:11:01.004868+0000 mgr.y (mgr.24287) 102 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:02.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:02 smithi148 bash[31149]: cluster 2024-03-07T03:11:01.004868+0000 mgr.y (mgr.24287) 102 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:02.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:02 smithi195 bash[23221]: cluster 2024-03-07T03:11:01.004868+0000 mgr.y (mgr.24287) 102 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:04.634 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:04 smithi148 bash[21362]: cluster 2024-03-07T03:11:03.005493+0000 mgr.y (mgr.24287) 103 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:04.635 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:04 smithi148 bash[21362]: audit 2024-03-07T03:11:04.046168+0000 mon.b (mon.1) 82 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:04.635 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:04 smithi148 bash[31149]: cluster 2024-03-07T03:11:03.005493+0000 mgr.y (mgr.24287) 103 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:04.635 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:04 smithi148 bash[31149]: audit 2024-03-07T03:11:04.046168+0000 mon.b (mon.1) 82 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:04.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:04 smithi195 bash[23221]: cluster 2024-03-07T03:11:03.005493+0000 mgr.y (mgr.24287) 103 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:04.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:04 smithi195 bash[23221]: audit 2024-03-07T03:11:04.046168+0000 mon.b (mon.1) 82 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:04.990 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:05.014 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:04 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:11:04] "GET /metrics HTTP/1.1" 200 28562 "" "Prometheus/2.43.0" 2024-03-07T03:11:06.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:06 smithi148 bash[21362]: cluster 2024-03-07T03:11:05.006082+0000 mgr.y (mgr.24287) 104 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:06.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:06 smithi148 bash[31149]: cluster 2024-03-07T03:11:05.006082+0000 mgr.y (mgr.24287) 104 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:06.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:06 smithi195 bash[23221]: cluster 2024-03-07T03:11:05.006082+0000 mgr.y (mgr.24287) 104 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:07.707 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:11:07.707 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-07T03:11:08.307 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-07T03:11:08.307 INFO:tasks.cephadm:Setup complete, yielding 2024-03-07T03:11:08.307 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-07T03:11:08.317 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi148.front.sepia.ceph.com 2024-03-07T03:11:08.317 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- bash -c 'ceph fs volume create foo' 2024-03-07T03:11:08.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:08 smithi148 bash[21362]: cluster 2024-03-07T03:11:07.006585+0000 mgr.y (mgr.24287) 105 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:08.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:08 smithi148 bash[21362]: audit 2024-03-07T03:11:07.718273+0000 mon.a (mon.0) 760 : audit 0 from='client.? 172.21.15.148:0/299802619' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:11:08.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:08 smithi148 bash[31149]: cluster 2024-03-07T03:11:07.006585+0000 mgr.y (mgr.24287) 105 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:08.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:08 smithi148 bash[31149]: audit 2024-03-07T03:11:07.718273+0000 mon.a (mon.0) 760 : audit 0 from='client.? 172.21.15.148:0/299802619' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:11:08.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:08 smithi195 bash[23221]: cluster 2024-03-07T03:11:07.006585+0000 mgr.y (mgr.24287) 105 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:08.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:08 smithi195 bash[23221]: audit 2024-03-07T03:11:07.718273+0000 mon.a (mon.0) 760 : audit 0 from='client.? 172.21.15.148:0/299802619' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:11:10.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:10 smithi148 bash[21362]: cluster 2024-03-07T03:11:09.007154+0000 mgr.y (mgr.24287) 106 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:10.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:10 smithi148 bash[31149]: cluster 2024-03-07T03:11:09.007154+0000 mgr.y (mgr.24287) 106 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:10.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:10 smithi195 bash[23221]: cluster 2024-03-07T03:11:09.007154+0000 mgr.y (mgr.24287) 106 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:12.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[21362]: cluster 2024-03-07T03:11:11.007833+0000 mgr.y (mgr.24287) 107 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:12.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[21362]: audit 2024-03-07T03:11:11.728428+0000 mon.b (mon.1) 83 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:12.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[21362]: audit 2024-03-07T03:11:12.161668+0000 mon.a (mon.0) 761 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:12.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[21362]: audit 2024-03-07T03:11:12.179872+0000 mon.a (mon.0) 762 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:12.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[31149]: cluster 2024-03-07T03:11:11.007833+0000 mgr.y (mgr.24287) 107 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:12.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[31149]: audit 2024-03-07T03:11:11.728428+0000 mon.b (mon.1) 83 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:12.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[31149]: audit 2024-03-07T03:11:12.161668+0000 mon.a (mon.0) 761 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:12.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:12 smithi148 bash[31149]: audit 2024-03-07T03:11:12.179872+0000 mon.a (mon.0) 762 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:12.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:12 smithi195 bash[23221]: cluster 2024-03-07T03:11:11.007833+0000 mgr.y (mgr.24287) 107 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:12.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:12 smithi195 bash[23221]: audit 2024-03-07T03:11:11.728428+0000 mon.b (mon.1) 83 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:12.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:12 smithi195 bash[23221]: audit 2024-03-07T03:11:12.161668+0000 mon.a (mon.0) 761 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:12.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:12 smithi195 bash[23221]: audit 2024-03-07T03:11:12.179872+0000 mon.a (mon.0) 762 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:13.049 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:13.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:13 smithi148 bash[21362]: audit 2024-03-07T03:11:12.664888+0000 mon.b (mon.1) 84 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:13.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:13 smithi148 bash[21362]: audit 2024-03-07T03:11:12.666517+0000 mon.b (mon.1) 85 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:13.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:13 smithi148 bash[21362]: audit 2024-03-07T03:11:12.680686+0000 mon.a (mon.0) 763 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:13.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:13 smithi148 bash[31149]: audit 2024-03-07T03:11:12.664888+0000 mon.b (mon.1) 84 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:13.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:13 smithi148 bash[31149]: audit 2024-03-07T03:11:12.666517+0000 mon.b (mon.1) 85 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:13.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:13 smithi148 bash[31149]: audit 2024-03-07T03:11:12.680686+0000 mon.a (mon.0) 763 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:13.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:13 smithi195 bash[23221]: audit 2024-03-07T03:11:12.664888+0000 mon.b (mon.1) 84 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:13.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:13 smithi195 bash[23221]: audit 2024-03-07T03:11:12.666517+0000 mon.b (mon.1) 85 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:13.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:13 smithi195 bash[23221]: audit 2024-03-07T03:11:12.680686+0000 mon.a (mon.0) 763 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:15.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:14 smithi148 bash[21362]: cluster 2024-03-07T03:11:13.008391+0000 mgr.y (mgr.24287) 108 : cluster 0 pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:15.043 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:14 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:11:14] "GET /metrics HTTP/1.1" 200 28562 "" "Prometheus/2.43.0" 2024-03-07T03:11:15.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:14 smithi148 bash[31149]: cluster 2024-03-07T03:11:13.008391+0000 mgr.y (mgr.24287) 108 : cluster 0 pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:15.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:14 smithi195 bash[23221]: cluster 2024-03-07T03:11:13.008391+0000 mgr.y (mgr.24287) 108 : cluster 0 pgmap v79: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:17.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[21362]: cluster 2024-03-07T03:11:15.008994+0000 mgr.y (mgr.24287) 109 : cluster 0 pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:17.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[21362]: audit 2024-03-07T03:11:15.837881+0000 mon.b (mon.1) 86 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:11:17.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[21362]: audit 2024-03-07T03:11:15.839319+0000 mgr.y (mgr.24287) 110 : audit 0 from='client.24476 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:17.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[21362]: audit 2024-03-07T03:11:15.841372+0000 mon.a (mon.0) 764 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:11:17.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[31149]: cluster 2024-03-07T03:11:15.008994+0000 mgr.y (mgr.24287) 109 : cluster 0 pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:17.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[31149]: audit 2024-03-07T03:11:15.837881+0000 mon.b (mon.1) 86 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:11:17.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[31149]: audit 2024-03-07T03:11:15.839319+0000 mgr.y (mgr.24287) 110 : audit 0 from='client.24476 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:17.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[31149]: audit 2024-03-07T03:11:15.841372+0000 mon.a (mon.0) 764 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:11:17.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:17 smithi195 bash[23221]: cluster 2024-03-07T03:11:15.008994+0000 mgr.y (mgr.24287) 109 : cluster 0 pgmap v80: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:17.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:17 smithi195 bash[23221]: audit 2024-03-07T03:11:15.837881+0000 mon.b (mon.1) 86 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:11:17.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:17 smithi195 bash[23221]: audit 2024-03-07T03:11:15.839319+0000 mgr.y (mgr.24287) 110 : audit 0 from='client.24476 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:11:17.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:17 smithi195 bash[23221]: audit 2024-03-07T03:11:15.841372+0000 mon.a (mon.0) 764 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-07T03:11:17.971 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:17 smithi148 bash[21362]: debug 2024-03-07T03:11:17.718+0000 7f751587b700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:11:18.245 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:16.698844+0000 mon.a (mon.0) 765 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-07T03:11:18.245 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cluster 2024-03-07T03:11:16.699052+0000 mon.a (mon.0) 766 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-03-07T03:11:18.245 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:16.702133+0000 mon.b (mon.1) 87 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.245 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:16.708014+0000 mon.a (mon.0) 767 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.245 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cluster 2024-03-07T03:11:17.009534+0000 mgr.y (mgr.24287) 111 : cluster 0 pgmap v82: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:18.246 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.703439+0000 mon.a (mon.0) 768 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-07T03:11:18.246 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cluster 2024-03-07T03:11:17.703747+0000 mon.a (mon.0) 769 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-03-07T03:11:18.246 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.712975+0000 mon.b (mon.1) 88 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.246 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.719543+0000 mon.a (mon.0) 770 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.246 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cluster 2024-03-07T03:11:17.721810+0000 mon.a (mon.0) 771 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:11:18.246 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cluster 2024-03-07T03:11:17.722505+0000 mon.a (mon.0) 772 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-07T03:11:18.246 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cluster 2024-03-07T03:11:17.744282+0000 mon.a (mon.0) 773 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.744603+0000 mon.a (mon.0) 774 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cluster 2024-03-07T03:11:17.745016+0000 mon.a (mon.0) 775 : cluster 0 fsmap foo:0 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cephadm 2024-03-07T03:11:17.752047+0000 mgr.y (mgr.24287) 112 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.768961+0000 mon.a (mon.0) 776 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.789602+0000 mon.b (mon.1) 89 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.791622+0000 mon.b (mon.1) 90 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.792448+0000 mon.b (mon.1) 91 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.800145+0000 mon.a (mon.0) 777 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:18.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.803139+0000 mon.b (mon.1) 92 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.806017+0000 mon.a (mon.0) 778 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.808764+0000 mon.b (mon.1) 93 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: audit 2024-03-07T03:11:17.810001+0000 mon.a (mon.0) 779 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[21362]: cephadm 2024-03-07T03:11:17.812422+0000 mgr.y (mgr.24287) 113 : cephadm 1 Deploying daemon mds.foo.smithi148.qizikc on smithi148 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:16.698844+0000 mon.a (mon.0) 765 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cluster 2024-03-07T03:11:16.699052+0000 mon.a (mon.0) 766 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:16.702133+0000 mon.b (mon.1) 87 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:16.708014+0000 mon.a (mon.0) 767 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cluster 2024-03-07T03:11:17.009534+0000 mgr.y (mgr.24287) 111 : cluster 0 pgmap v82: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.703439+0000 mon.a (mon.0) 768 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cluster 2024-03-07T03:11:17.703747+0000 mon.a (mon.0) 769 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.712975+0000 mon.b (mon.1) 88 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.719543+0000 mon.a (mon.0) 770 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cluster 2024-03-07T03:11:17.721810+0000 mon.a (mon.0) 771 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cluster 2024-03-07T03:11:17.722505+0000 mon.a (mon.0) 772 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cluster 2024-03-07T03:11:17.744282+0000 mon.a (mon.0) 773 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.744603+0000 mon.a (mon.0) 774 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cluster 2024-03-07T03:11:17.745016+0000 mon.a (mon.0) 775 : cluster 0 fsmap foo:0 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cephadm 2024-03-07T03:11:17.752047+0000 mgr.y (mgr.24287) 112 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.768961+0000 mon.a (mon.0) 776 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.789602+0000 mon.b (mon.1) 89 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:18.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.791622+0000 mon.b (mon.1) 90 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:18.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.792448+0000 mon.b (mon.1) 91 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:18.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.800145+0000 mon.a (mon.0) 777 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:18.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.803139+0000 mon.b (mon.1) 92 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:18.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.806017+0000 mon.a (mon.0) 778 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:18.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.808764+0000 mon.b (mon.1) 93 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:18.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: audit 2024-03-07T03:11:17.810001+0000 mon.a (mon.0) 779 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:11:18.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:18 smithi148 bash[31149]: cephadm 2024-03-07T03:11:17.812422+0000 mgr.y (mgr.24287) 113 : cephadm 1 Deploying daemon mds.foo.smithi148.qizikc on smithi148 2024-03-07T03:11:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:16.698844+0000 mon.a (mon.0) 765 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-07T03:11:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cluster 2024-03-07T03:11:16.699052+0000 mon.a (mon.0) 766 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:16.702133+0000 mon.b (mon.1) 87 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:16.708014+0000 mon.a (mon.0) 767 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cluster 2024-03-07T03:11:17.009534+0000 mgr.y (mgr.24287) 111 : cluster 0 pgmap v82: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.703439+0000 mon.a (mon.0) 768 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cluster 2024-03-07T03:11:17.703747+0000 mon.a (mon.0) 769 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.712975+0000 mon.b (mon.1) 88 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.719543+0000 mon.a (mon.0) 770 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cluster 2024-03-07T03:11:17.721810+0000 mon.a (mon.0) 771 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cluster 2024-03-07T03:11:17.722505+0000 mon.a (mon.0) 772 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cluster 2024-03-07T03:11:17.744282+0000 mon.a (mon.0) 773 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.744603+0000 mon.a (mon.0) 774 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cluster 2024-03-07T03:11:17.745016+0000 mon.a (mon.0) 775 : cluster 0 fsmap foo:0 2024-03-07T03:11:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cephadm 2024-03-07T03:11:17.752047+0000 mgr.y (mgr.24287) 112 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.768961+0000 mon.a (mon.0) 776 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.789602+0000 mon.b (mon.1) 89 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.791622+0000 mon.b (mon.1) 90 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.792448+0000 mon.b (mon.1) 91 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.800145+0000 mon.a (mon.0) 777 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.803139+0000 mon.b (mon.1) 92 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.806017+0000 mon.a (mon.0) 778 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.808764+0000 mon.b (mon.1) 93 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: audit 2024-03-07T03:11:17.810001+0000 mon.a (mon.0) 779 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:11:18.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:18 smithi195 bash[23221]: cephadm 2024-03-07T03:11:17.812422+0000 mgr.y (mgr.24287) 113 : cephadm 1 Deploying daemon mds.foo.smithi148.qizikc on smithi148 2024-03-07T03:11:19.028 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-07T03:11:19.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[21362]: cluster 2024-03-07T03:11:18.121938+0000 mon.a (mon.0) 780 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:19.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[21362]: cluster 2024-03-07T03:11:18.738941+0000 mon.a (mon.0) 781 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-03-07T03:11:19.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[21362]: audit 2024-03-07T03:11:19.054416+0000 mon.a (mon.0) 782 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:19.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[21362]: audit 2024-03-07T03:11:19.056347+0000 mon.b (mon.1) 94 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:19.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[31149]: cluster 2024-03-07T03:11:18.121938+0000 mon.a (mon.0) 780 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:19.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[31149]: cluster 2024-03-07T03:11:18.738941+0000 mon.a (mon.0) 781 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-03-07T03:11:19.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[31149]: audit 2024-03-07T03:11:19.054416+0000 mon.a (mon.0) 782 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:19.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:19 smithi148 bash[31149]: audit 2024-03-07T03:11:19.056347+0000 mon.b (mon.1) 94 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:19.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:19 smithi195 bash[23221]: cluster 2024-03-07T03:11:18.121938+0000 mon.a (mon.0) 780 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:11:19.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:19 smithi195 bash[23221]: cluster 2024-03-07T03:11:18.738941+0000 mon.a (mon.0) 781 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-03-07T03:11:19.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:19 smithi195 bash[23221]: audit 2024-03-07T03:11:19.054416+0000 mon.a (mon.0) 782 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:19.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:19 smithi195 bash[23221]: audit 2024-03-07T03:11:19.056347+0000 mon.b (mon.1) 94 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:20.438 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.439 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.439 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.439 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.440 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:20 smithi148 bash[21362]: cluster 2024-03-07T03:11:19.010183+0000 mgr.y (mgr.24287) 114 : cluster 0 pgmap v86: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:20.440 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:20 smithi148 bash[21362]: cluster 2024-03-07T03:11:19.731448+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:20.440 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:20 smithi148 bash[21362]: cluster 2024-03-07T03:11:19.749370+0000 mon.a (mon.0) 784 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-03-07T03:11:20.440 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.440 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.441 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.441 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:20 smithi148 bash[31149]: cluster 2024-03-07T03:11:19.010183+0000 mgr.y (mgr.24287) 114 : cluster 0 pgmap v86: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:20.441 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:20 smithi148 bash[31149]: cluster 2024-03-07T03:11:19.731448+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:20.441 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:20 smithi148 bash[31149]: cluster 2024-03-07T03:11:19.749370+0000 mon.a (mon.0) 784 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-03-07T03:11:20.441 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.441 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:20 smithi195 bash[23221]: cluster 2024-03-07T03:11:19.010183+0000 mgr.y (mgr.24287) 114 : cluster 0 pgmap v86: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:20.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:20 smithi195 bash[23221]: cluster 2024-03-07T03:11:19.731448+0000 mon.a (mon.0) 783 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:11:20.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:20 smithi195 bash[23221]: cluster 2024-03-07T03:11:19.749370+0000 mon.a (mon.0) 784 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-03-07T03:11:20.690 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.690 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.691 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.691 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.691 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.691 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.692 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.692 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:20.692 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:22.009 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: audit 2024-03-07T03:11:20.737026+0000 mon.a (mon.0) 785 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.009 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: audit 2024-03-07T03:11:20.749103+0000 mon.a (mon.0) 786 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.009 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: audit 2024-03-07T03:11:20.787939+0000 mon.a (mon.0) 787 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.010 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: audit 2024-03-07T03:11:20.789347+0000 mon.b (mon.1) 95 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:22.010 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: audit 2024-03-07T03:11:20.792482+0000 mon.a (mon.0) 788 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:22.010 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: audit 2024-03-07T03:11:20.795818+0000 mon.a (mon.0) 789 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:11:22.010 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: audit 2024-03-07T03:11:20.797392+0000 mon.b (mon.1) 96 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:22.010 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:21 smithi195 bash[23221]: cephadm 2024-03-07T03:11:20.801191+0000 mgr.y (mgr.24287) 115 : cephadm 1 Deploying daemon mds.foo.smithi195.gtlmjn on smithi195 2024-03-07T03:11:22.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: audit 2024-03-07T03:11:20.737026+0000 mon.a (mon.0) 785 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: audit 2024-03-07T03:11:20.749103+0000 mon.a (mon.0) 786 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: audit 2024-03-07T03:11:20.787939+0000 mon.a (mon.0) 787 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: audit 2024-03-07T03:11:20.789347+0000 mon.b (mon.1) 95 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: audit 2024-03-07T03:11:20.792482+0000 mon.a (mon.0) 788 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: audit 2024-03-07T03:11:20.795818+0000 mon.a (mon.0) 789 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: audit 2024-03-07T03:11:20.797392+0000 mon.b (mon.1) 96 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[21362]: cephadm 2024-03-07T03:11:20.801191+0000 mgr.y (mgr.24287) 115 : cephadm 1 Deploying daemon mds.foo.smithi195.gtlmjn on smithi195 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: audit 2024-03-07T03:11:20.737026+0000 mon.a (mon.0) 785 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: audit 2024-03-07T03:11:20.749103+0000 mon.a (mon.0) 786 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: audit 2024-03-07T03:11:20.787939+0000 mon.a (mon.0) 787 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: audit 2024-03-07T03:11:20.789347+0000 mon.b (mon.1) 95 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: audit 2024-03-07T03:11:20.792482+0000 mon.a (mon.0) 788 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: audit 2024-03-07T03:11:20.795818+0000 mon.a (mon.0) 789 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: audit 2024-03-07T03:11:20.797392+0000 mon.b (mon.1) 96 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:22.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:21 smithi148 bash[31149]: cephadm 2024-03-07T03:11:20.801191+0000 mgr.y (mgr.24287) 115 : cephadm 1 Deploying daemon mds.foo.smithi195.gtlmjn on smithi195 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.011094+0000 mgr.y (mgr.24287) 116 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: audit 2024-03-07T03:11:21.802622+0000 mon.b (mon.1) 97 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.802920+0000 mon.a (mon.0) 790 : cluster 0 mds.? [v2:172.21.15.148:6834/1878290118,v1:172.21.15.148:6835/1878290118] up:boot 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.802980+0000 mon.a (mon.0) 791 : cluster 1 daemon mds.foo.smithi148.qizikc assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.803049+0000 mon.a (mon.0) 792 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.803072+0000 mon.a (mon.0) 793 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.803089+0000 mon.a (mon.0) 794 : cluster 1 Cluster is now healthy 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.803339+0000 mon.a (mon.0) 795 : cluster 0 fsmap foo:0 1 up:standby 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.807403+0000 mon.a (mon.0) 796 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:creating} 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[21362]: cluster 2024-03-07T03:11:21.835408+0000 mon.a (mon.0) 797 : cluster 1 daemon mds.foo.smithi148.qizikc is now active in filesystem foo as rank 0 2024-03-07T03:11:23.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.011094+0000 mgr.y (mgr.24287) 116 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: audit 2024-03-07T03:11:21.802622+0000 mon.b (mon.1) 97 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.802920+0000 mon.a (mon.0) 790 : cluster 0 mds.? [v2:172.21.15.148:6834/1878290118,v1:172.21.15.148:6835/1878290118] up:boot 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.802980+0000 mon.a (mon.0) 791 : cluster 1 daemon mds.foo.smithi148.qizikc assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.803049+0000 mon.a (mon.0) 792 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.803072+0000 mon.a (mon.0) 793 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.803089+0000 mon.a (mon.0) 794 : cluster 1 Cluster is now healthy 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.803339+0000 mon.a (mon.0) 795 : cluster 0 fsmap foo:0 1 up:standby 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.807403+0000 mon.a (mon.0) 796 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:creating} 2024-03-07T03:11:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:22 smithi148 bash[31149]: cluster 2024-03-07T03:11:21.835408+0000 mon.a (mon.0) 797 : cluster 1 daemon mds.foo.smithi148.qizikc is now active in filesystem foo as rank 0 2024-03-07T03:11:23.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.011094+0000 mgr.y (mgr.24287) 116 : cluster 0 pgmap v88: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: audit 2024-03-07T03:11:21.802622+0000 mon.b (mon.1) 97 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.802920+0000 mon.a (mon.0) 790 : cluster 0 mds.? [v2:172.21.15.148:6834/1878290118,v1:172.21.15.148:6835/1878290118] up:boot 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.802980+0000 mon.a (mon.0) 791 : cluster 1 daemon mds.foo.smithi148.qizikc assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.803049+0000 mon.a (mon.0) 792 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.803072+0000 mon.a (mon.0) 793 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.803089+0000 mon.a (mon.0) 794 : cluster 1 Cluster is now healthy 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.803339+0000 mon.a (mon.0) 795 : cluster 0 fsmap foo:0 1 up:standby 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.807403+0000 mon.a (mon.0) 796 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:creating} 2024-03-07T03:11:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:22 smithi195 bash[23221]: cluster 2024-03-07T03:11:21.835408+0000 mon.a (mon.0) 797 : cluster 1 daemon mds.foo.smithi148.qizikc is now active in filesystem foo as rank 0 2024-03-07T03:11:23.367 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.367 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.368 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.368 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.368 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.369 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.369 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.369 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.369 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.369 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.761 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:23.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.812 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.812 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.812 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.813 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.813 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:23.813 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:11:23 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:11:24.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: cluster 2024-03-07T03:11:22.819794+0000 mon.a (mon.0) 798 : cluster 0 mds.? [v2:172.21.15.148:6834/1878290118,v1:172.21.15.148:6835/1878290118] up:active 2024-03-07T03:11:24.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: cluster 2024-03-07T03:11:22.819959+0000 mon.a (mon.0) 799 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:active} 2024-03-07T03:11:24.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: audit 2024-03-07T03:11:23.543602+0000 mon.a (mon.0) 800 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: audit 2024-03-07T03:11:23.559285+0000 mon.a (mon.0) 801 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: audit 2024-03-07T03:11:23.581024+0000 mon.a (mon.0) 802 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: audit 2024-03-07T03:11:23.593609+0000 mon.a (mon.0) 803 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: audit 2024-03-07T03:11:23.604586+0000 mon.a (mon.0) 804 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[21362]: audit 2024-03-07T03:11:23.668659+0000 mon.b (mon.1) 98 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: cluster 2024-03-07T03:11:22.819794+0000 mon.a (mon.0) 798 : cluster 0 mds.? [v2:172.21.15.148:6834/1878290118,v1:172.21.15.148:6835/1878290118] up:active 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: cluster 2024-03-07T03:11:22.819959+0000 mon.a (mon.0) 799 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:active} 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: audit 2024-03-07T03:11:23.543602+0000 mon.a (mon.0) 800 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: audit 2024-03-07T03:11:23.559285+0000 mon.a (mon.0) 801 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: audit 2024-03-07T03:11:23.581024+0000 mon.a (mon.0) 802 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: audit 2024-03-07T03:11:23.593609+0000 mon.a (mon.0) 803 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: audit 2024-03-07T03:11:23.604586+0000 mon.a (mon.0) 804 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:23 smithi148 bash[31149]: audit 2024-03-07T03:11:23.668659+0000 mon.b (mon.1) 98 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: cluster 2024-03-07T03:11:22.819794+0000 mon.a (mon.0) 798 : cluster 0 mds.? [v2:172.21.15.148:6834/1878290118,v1:172.21.15.148:6835/1878290118] up:active 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: cluster 2024-03-07T03:11:22.819959+0000 mon.a (mon.0) 799 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:active} 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: audit 2024-03-07T03:11:23.543602+0000 mon.a (mon.0) 800 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: audit 2024-03-07T03:11:23.559285+0000 mon.a (mon.0) 801 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: audit 2024-03-07T03:11:23.581024+0000 mon.a (mon.0) 802 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: audit 2024-03-07T03:11:23.593609+0000 mon.a (mon.0) 803 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: audit 2024-03-07T03:11:23.604586+0000 mon.a (mon.0) 804 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:23 smithi195 bash[23221]: audit 2024-03-07T03:11:23.668659+0000 mon.b (mon.1) 98 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:24.885 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:24 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:11:24] "GET /metrics HTTP/1.1" 200 33361 "" "Prometheus/2.43.0" 2024-03-07T03:11:24.885 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:24 smithi148 bash[21362]: cluster 2024-03-07T03:11:23.011749+0000 mgr.y (mgr.24287) 117 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:24.885 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:24 smithi148 bash[21362]: audit 2024-03-07T03:11:24.121405+0000 mon.a (mon.0) 805 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:24.885 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:24 smithi148 bash[31149]: cluster 2024-03-07T03:11:23.011749+0000 mgr.y (mgr.24287) 117 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:24.886 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:24 smithi148 bash[31149]: audit 2024-03-07T03:11:24.121405+0000 mon.a (mon.0) 805 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:25.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:24 smithi195 bash[23221]: cluster 2024-03-07T03:11:23.011749+0000 mgr.y (mgr.24287) 117 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:25.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:24 smithi195 bash[23221]: audit 2024-03-07T03:11:24.121405+0000 mon.a (mon.0) 805 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.012860+0000 mgr.y (mgr.24287) 118 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-03-07T03:11:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: audit 2024-03-07T03:11:25.122219+0000 mon.b (mon.1) 99 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:11:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.123495+0000 mon.a (mon.0) 806 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:boot 2024-03-07T03:11:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.123631+0000 mon.a (mon.0) 807 : cluster 1 Dropping low affinity active daemon mds.foo.smithi148.qizikc in favor of higher affinity standby. 2024-03-07T03:11:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.123714+0000 mon.a (mon.0) 808 : cluster 3 Replacing daemon mds.foo.smithi148.qizikc as rank 0 with standby daemon mds.foo.smithi195.gtlmjn 2024-03-07T03:11:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.125053+0000 mon.a (mon.0) 809 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-07T03:11:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.125119+0000 mon.a (mon.0) 810 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.125196+0000 mon.a (mon.0) 811 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:active} 1 up:standby 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.132709+0000 mon.a (mon.0) 812 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[21362]: cluster 2024-03-07T03:11:25.142896+0000 mon.a (mon.0) 813 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:replay} 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.012860+0000 mgr.y (mgr.24287) 118 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: audit 2024-03-07T03:11:25.122219+0000 mon.b (mon.1) 99 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.123495+0000 mon.a (mon.0) 806 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:boot 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.123631+0000 mon.a (mon.0) 807 : cluster 1 Dropping low affinity active daemon mds.foo.smithi148.qizikc in favor of higher affinity standby. 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.123714+0000 mon.a (mon.0) 808 : cluster 3 Replacing daemon mds.foo.smithi148.qizikc as rank 0 with standby daemon mds.foo.smithi195.gtlmjn 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.125053+0000 mon.a (mon.0) 809 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.125119+0000 mon.a (mon.0) 810 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.125196+0000 mon.a (mon.0) 811 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:active} 1 up:standby 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.132709+0000 mon.a (mon.0) 812 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-03-07T03:11:26.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:26 smithi148 bash[31149]: cluster 2024-03-07T03:11:25.142896+0000 mon.a (mon.0) 813 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:replay} 2024-03-07T03:11:26.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.012860+0000 mgr.y (mgr.24287) 118 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s wr, 6 op/s 2024-03-07T03:11:26.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: audit 2024-03-07T03:11:25.122219+0000 mon.b (mon.1) 99 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:11:26.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.123495+0000 mon.a (mon.0) 806 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:boot 2024-03-07T03:11:26.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.123631+0000 mon.a (mon.0) 807 : cluster 1 Dropping low affinity active daemon mds.foo.smithi148.qizikc in favor of higher affinity standby. 2024-03-07T03:11:26.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.123714+0000 mon.a (mon.0) 808 : cluster 3 Replacing daemon mds.foo.smithi148.qizikc as rank 0 with standby daemon mds.foo.smithi195.gtlmjn 2024-03-07T03:11:26.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.125053+0000 mon.a (mon.0) 809 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-03-07T03:11:26.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.125119+0000 mon.a (mon.0) 810 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-03-07T03:11:26.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.125196+0000 mon.a (mon.0) 811 : cluster 0 fsmap foo:1 {0=foo.smithi148.qizikc=up:active} 1 up:standby 2024-03-07T03:11:26.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.132709+0000 mon.a (mon.0) 812 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-03-07T03:11:26.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:26 smithi195 bash[23221]: cluster 2024-03-07T03:11:25.142896+0000 mon.a (mon.0) 813 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:replay} 2024-03-07T03:11:27.313 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-07T03:11:27.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[21362]: cluster 2024-03-07T03:11:26.135686+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-07T03:11:27.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[21362]: audit 2024-03-07T03:11:26.140122+0000 mon.b (mon.1) 100 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:11:27.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[21362]: cluster 2024-03-07T03:11:26.142448+0000 mon.a (mon.0) 815 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:reconnect 2024-03-07T03:11:27.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[21362]: cluster 2024-03-07T03:11:26.142566+0000 mon.a (mon.0) 816 : cluster 0 mds.? [v2:172.21.15.148:6834/4101454543,v1:172.21.15.148:6835/4101454543] up:boot 2024-03-07T03:11:27.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[21362]: cluster 2024-03-07T03:11:26.142720+0000 mon.a (mon.0) 817 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:reconnect} 1 up:standby 2024-03-07T03:11:27.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[31149]: cluster 2024-03-07T03:11:26.135686+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-07T03:11:27.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[31149]: audit 2024-03-07T03:11:26.140122+0000 mon.b (mon.1) 100 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:11:27.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[31149]: cluster 2024-03-07T03:11:26.142448+0000 mon.a (mon.0) 815 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:reconnect 2024-03-07T03:11:27.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[31149]: cluster 2024-03-07T03:11:26.142566+0000 mon.a (mon.0) 816 : cluster 0 mds.? [v2:172.21.15.148:6834/4101454543,v1:172.21.15.148:6835/4101454543] up:boot 2024-03-07T03:11:27.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:27 smithi148 bash[31149]: cluster 2024-03-07T03:11:26.142720+0000 mon.a (mon.0) 817 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:reconnect} 1 up:standby 2024-03-07T03:11:27.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:27 smithi195 bash[23221]: cluster 2024-03-07T03:11:26.135686+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-03-07T03:11:27.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:27 smithi195 bash[23221]: audit 2024-03-07T03:11:26.140122+0000 mon.b (mon.1) 100 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:11:27.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:27 smithi195 bash[23221]: cluster 2024-03-07T03:11:26.142448+0000 mon.a (mon.0) 815 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:reconnect 2024-03-07T03:11:27.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:27 smithi195 bash[23221]: cluster 2024-03-07T03:11:26.142566+0000 mon.a (mon.0) 816 : cluster 0 mds.? [v2:172.21.15.148:6834/4101454543,v1:172.21.15.148:6835/4101454543] up:boot 2024-03-07T03:11:27.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:27 smithi195 bash[23221]: cluster 2024-03-07T03:11:26.142720+0000 mon.a (mon.0) 817 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:reconnect} 1 up:standby 2024-03-07T03:11:28.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[21362]: cluster 2024-03-07T03:11:27.013579+0000 mgr.y (mgr.24287) 119 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-03-07T03:11:28.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[21362]: cluster 2024-03-07T03:11:27.142307+0000 mon.a (mon.0) 818 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:rejoin 2024-03-07T03:11:28.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[21362]: cluster 2024-03-07T03:11:27.142382+0000 mon.a (mon.0) 819 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:rejoin} 1 up:standby 2024-03-07T03:11:28.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[21362]: cluster 2024-03-07T03:11:27.144304+0000 mon.a (mon.0) 820 : cluster 1 daemon mds.foo.smithi195.gtlmjn is now active in filesystem foo as rank 0 2024-03-07T03:11:28.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[31149]: cluster 2024-03-07T03:11:27.013579+0000 mgr.y (mgr.24287) 119 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-03-07T03:11:28.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[31149]: cluster 2024-03-07T03:11:27.142307+0000 mon.a (mon.0) 818 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:rejoin 2024-03-07T03:11:28.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[31149]: cluster 2024-03-07T03:11:27.142382+0000 mon.a (mon.0) 819 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:rejoin} 1 up:standby 2024-03-07T03:11:28.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:28 smithi148 bash[31149]: cluster 2024-03-07T03:11:27.144304+0000 mon.a (mon.0) 820 : cluster 1 daemon mds.foo.smithi195.gtlmjn is now active in filesystem foo as rank 0 2024-03-07T03:11:28.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:28 smithi195 bash[23221]: cluster 2024-03-07T03:11:27.013579+0000 mgr.y (mgr.24287) 119 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-03-07T03:11:28.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:28 smithi195 bash[23221]: cluster 2024-03-07T03:11:27.142307+0000 mon.a (mon.0) 818 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:rejoin 2024-03-07T03:11:28.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:28 smithi195 bash[23221]: cluster 2024-03-07T03:11:27.142382+0000 mon.a (mon.0) 819 : cluster 0 fsmap foo:1/1 {0=foo.smithi195.gtlmjn=up:rejoin} 1 up:standby 2024-03-07T03:11:28.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:28 smithi195 bash[23221]: cluster 2024-03-07T03:11:27.144304+0000 mon.a (mon.0) 820 : cluster 1 daemon mds.foo.smithi195.gtlmjn is now active in filesystem foo as rank 0 2024-03-07T03:11:29.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[31149]: cluster 2024-03-07T03:11:28.139284+0000 mon.a (mon.0) 821 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-07T03:11:29.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[31149]: cluster 2024-03-07T03:11:28.139311+0000 mon.a (mon.0) 822 : cluster 1 Cluster is now healthy 2024-03-07T03:11:29.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[31149]: cluster 2024-03-07T03:11:28.143914+0000 mon.a (mon.0) 823 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:active 2024-03-07T03:11:29.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[31149]: cluster 2024-03-07T03:11:28.144015+0000 mon.a (mon.0) 824 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:11:29.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[31149]: audit 2024-03-07T03:11:28.459212+0000 mon.a (mon.0) 825 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[31149]: audit 2024-03-07T03:11:28.471719+0000 mon.a (mon.0) 826 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[31149]: audit 2024-03-07T03:11:29.133413+0000 mon.a (mon.0) 827 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[21362]: cluster 2024-03-07T03:11:28.139284+0000 mon.a (mon.0) 821 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-07T03:11:29.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[21362]: cluster 2024-03-07T03:11:28.139311+0000 mon.a (mon.0) 822 : cluster 1 Cluster is now healthy 2024-03-07T03:11:29.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[21362]: cluster 2024-03-07T03:11:28.143914+0000 mon.a (mon.0) 823 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:active 2024-03-07T03:11:29.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[21362]: cluster 2024-03-07T03:11:28.144015+0000 mon.a (mon.0) 824 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:11:29.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[21362]: audit 2024-03-07T03:11:28.459212+0000 mon.a (mon.0) 825 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[21362]: audit 2024-03-07T03:11:28.471719+0000 mon.a (mon.0) 826 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:29 smithi148 bash[21362]: audit 2024-03-07T03:11:29.133413+0000 mon.a (mon.0) 827 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:29 smithi195 bash[23221]: cluster 2024-03-07T03:11:28.139284+0000 mon.a (mon.0) 821 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-03-07T03:11:29.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:29 smithi195 bash[23221]: cluster 2024-03-07T03:11:28.139311+0000 mon.a (mon.0) 822 : cluster 1 Cluster is now healthy 2024-03-07T03:11:29.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:29 smithi195 bash[23221]: cluster 2024-03-07T03:11:28.143914+0000 mon.a (mon.0) 823 : cluster 0 mds.? [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] up:active 2024-03-07T03:11:29.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:29 smithi195 bash[23221]: cluster 2024-03-07T03:11:28.144015+0000 mon.a (mon.0) 824 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:11:29.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:29 smithi195 bash[23221]: audit 2024-03-07T03:11:28.459212+0000 mon.a (mon.0) 825 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:29 smithi195 bash[23221]: audit 2024-03-07T03:11:28.471719+0000 mon.a (mon.0) 826 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:29.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:29 smithi195 bash[23221]: audit 2024-03-07T03:11:29.133413+0000 mon.a (mon.0) 827 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: cluster 2024-03-07T03:11:29.014356+0000 mgr.y (mgr.24287) 120 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-07T03:11:30.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.423036+0000 mon.a (mon.0) 828 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.439160+0000 mon.a (mon.0) 829 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.440855+0000 mon.b (mon.1) 101 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.442835+0000 mon.b (mon.1) 102 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.456720+0000 mon.a (mon.0) 830 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.535801+0000 mon.b (mon.1) 103 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.539189+0000 mon.b (mon.1) 104 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.540986+0000 mon.b (mon.1) 105 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[31149]: audit 2024-03-07T03:11:29.554605+0000 mon.a (mon.0) 831 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: cluster 2024-03-07T03:11:29.014356+0000 mgr.y (mgr.24287) 120 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.423036+0000 mon.a (mon.0) 828 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.439160+0000 mon.a (mon.0) 829 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.440855+0000 mon.b (mon.1) 101 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.442835+0000 mon.b (mon.1) 102 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.456720+0000 mon.a (mon.0) 830 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.535801+0000 mon.b (mon.1) 103 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:30.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.539189+0000 mon.b (mon.1) 104 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:30.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.540986+0000 mon.b (mon.1) 105 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:30.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:30 smithi148 bash[21362]: audit 2024-03-07T03:11:29.554605+0000 mon.a (mon.0) 831 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: cluster 2024-03-07T03:11:29.014356+0000 mgr.y (mgr.24287) 120 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s wr, 4 op/s 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.423036+0000 mon.a (mon.0) 828 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.439160+0000 mon.a (mon.0) 829 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.440855+0000 mon.b (mon.1) 101 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.442835+0000 mon.b (mon.1) 102 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.456720+0000 mon.a (mon.0) 830 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.535801+0000 mon.b (mon.1) 103 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.539189+0000 mon.b (mon.1) 104 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.540986+0000 mon.b (mon.1) 105 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:11:30.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:30 smithi195 bash[23221]: audit 2024-03-07T03:11:29.554605+0000 mon.a (mon.0) 831 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:31.912 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:32.745 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:32 smithi148 bash[21362]: cluster 2024-03-07T03:11:31.015524+0000 mgr.y (mgr.24287) 121 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-03-07T03:11:32.745 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:32 smithi148 bash[31149]: cluster 2024-03-07T03:11:31.015524+0000 mgr.y (mgr.24287) 121 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-03-07T03:11:32.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:32 smithi195 bash[23221]: cluster 2024-03-07T03:11:31.015524+0000 mgr.y (mgr.24287) 121 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-03-07T03:11:34.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:34 smithi148 bash[21362]: cluster 2024-03-07T03:11:33.016171+0000 mgr.y (mgr.24287) 122 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-03-07T03:11:34.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:34 smithi148 bash[21362]: audit 2024-03-07T03:11:34.056250+0000 mon.b (mon.1) 106 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:34.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:34 smithi148 bash[21362]: audit 2024-03-07T03:11:34.056672+0000 mon.a (mon.0) 832 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:34.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:34 smithi148 bash[31149]: cluster 2024-03-07T03:11:33.016171+0000 mgr.y (mgr.24287) 122 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-03-07T03:11:34.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:34 smithi148 bash[31149]: audit 2024-03-07T03:11:34.056250+0000 mon.b (mon.1) 106 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:34.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:34 smithi148 bash[31149]: audit 2024-03-07T03:11:34.056672+0000 mon.a (mon.0) 832 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:34.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:34 smithi195 bash[23221]: cluster 2024-03-07T03:11:33.016171+0000 mgr.y (mgr.24287) 122 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-03-07T03:11:34.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:34 smithi195 bash[23221]: audit 2024-03-07T03:11:34.056250+0000 mon.b (mon.1) 106 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:34.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:34 smithi195 bash[23221]: audit 2024-03-07T03:11:34.056672+0000 mon.a (mon.0) 832 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:11:35.042 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:34 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:11:34] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2024-03-07T03:11:35.295 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-07T03:11:35.305 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-07T03:11:35.305 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-07T03:11:35.306 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph mgr dump --format=json 2024-03-07T03:11:36.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:36 smithi148 bash[21362]: cluster 2024-03-07T03:11:35.017213+0000 mgr.y (mgr.24287) 123 : cluster 0 pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-03-07T03:11:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:36 smithi148 bash[31149]: cluster 2024-03-07T03:11:35.017213+0000 mgr.y (mgr.24287) 123 : cluster 0 pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-03-07T03:11:36.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:36 smithi195 bash[23221]: cluster 2024-03-07T03:11:35.017213+0000 mgr.y (mgr.24287) 123 : cluster 0 pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-03-07T03:11:38.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:38 smithi148 bash[21362]: cluster 2024-03-07T03:11:37.017956+0000 mgr.y (mgr.24287) 124 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:38 smithi148 bash[31149]: cluster 2024-03-07T03:11:37.017956+0000 mgr.y (mgr.24287) 124 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:38.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:38 smithi195 bash[23221]: cluster 2024-03-07T03:11:37.017956+0000 mgr.y (mgr.24287) 124 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:40.035 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:40.786 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:40 smithi148 bash[21362]: cluster 2024-03-07T03:11:39.018746+0000 mgr.y (mgr.24287) 125 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:40.786 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:40 smithi148 bash[31149]: cluster 2024-03-07T03:11:39.018746+0000 mgr.y (mgr.24287) 125 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:40.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:40 smithi195 bash[23221]: cluster 2024-03-07T03:11:39.018746+0000 mgr.y (mgr.24287) 125 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:42.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:42 smithi148 bash[21362]: cluster 2024-03-07T03:11:41.020023+0000 mgr.y (mgr.24287) 126 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:42.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:42 smithi148 bash[31149]: cluster 2024-03-07T03:11:41.020023+0000 mgr.y (mgr.24287) 126 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:42.652 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:11:42.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:42 smithi195 bash[23221]: cluster 2024-03-07T03:11:41.020023+0000 mgr.y (mgr.24287) 126 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-03-07T03:11:43.285 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":20,"active_gid":24287,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6800","nonce":1790474515},{"type":"v1","addr":"172.21.15.148:6801","nonce":1790474515}]},"active_addr":"172.21.15.148:6801/1790474515","active_change":"2024-03-07T03:08:48.947088+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24298,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.148:8443/","prometheus":"http://172.21.15.148:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":2001636836}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":1314748318}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":3482682182}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":2057842025}]}]} 2024-03-07T03:11:43.290 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-07T03:11:43.290 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-07T03:11:43.290 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:11:43.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:43 smithi148 bash[21362]: audit 2024-03-07T03:11:42.656854+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.148:0/4124756567' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:11:43.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:43 smithi148 bash[31149]: audit 2024-03-07T03:11:42.656854+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.148:0/4124756567' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:11:43.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:43 smithi195 bash[23221]: audit 2024-03-07T03:11:42.656854+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.148:0/4124756567' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:11:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:44 smithi148 bash[21362]: cluster 2024-03-07T03:11:43.020680+0000 mgr.y (mgr.24287) 127 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:44.792 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:44 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:11:44] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2024-03-07T03:11:44.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:44 smithi148 bash[31149]: cluster 2024-03-07T03:11:43.020680+0000 mgr.y (mgr.24287) 127 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:44.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:44 smithi195 bash[23221]: cluster 2024-03-07T03:11:43.020680+0000 mgr.y (mgr.24287) 127 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:46.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:46 smithi148 bash[21362]: cluster 2024-03-07T03:11:45.021747+0000 mgr.y (mgr.24287) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:46.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:46 smithi148 bash[31149]: cluster 2024-03-07T03:11:45.021747+0000 mgr.y (mgr.24287) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:46.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:46 smithi195 bash[23221]: cluster 2024-03-07T03:11:45.021747+0000 mgr.y (mgr.24287) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:48.022 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:48.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:48 smithi148 bash[21362]: cluster 2024-03-07T03:11:47.022460+0000 mgr.y (mgr.24287) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:48.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:48 smithi148 bash[31149]: cluster 2024-03-07T03:11:47.022460+0000 mgr.y (mgr.24287) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:48.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:48 smithi195 bash[23221]: cluster 2024-03-07T03:11:47.022460+0000 mgr.y (mgr.24287) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[21362]: audit 2024-03-07T03:11:49.001253+0000 mon.b (mon.1) 107 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[21362]: audit 2024-03-07T03:11:49.001815+0000 mon.b (mon.1) 108 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[21362]: audit 2024-03-07T03:11:49.002289+0000 mon.b (mon.1) 109 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[21362]: audit 2024-03-07T03:11:49.004737+0000 mon.a (mon.0) 833 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[21362]: audit 2024-03-07T03:11:49.005023+0000 mon.a (mon.0) 834 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[21362]: audit 2024-03-07T03:11:49.005475+0000 mon.a (mon.0) 835 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[21362]: audit 2024-03-07T03:11:49.046382+0000 mon.b (mon.1) 110 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[31149]: audit 2024-03-07T03:11:49.001253+0000 mon.b (mon.1) 107 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[31149]: audit 2024-03-07T03:11:49.001815+0000 mon.b (mon.1) 108 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-03-07T03:11:49.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[31149]: audit 2024-03-07T03:11:49.002289+0000 mon.b (mon.1) 109 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[31149]: audit 2024-03-07T03:11:49.004737+0000 mon.a (mon.0) 833 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[31149]: audit 2024-03-07T03:11:49.005023+0000 mon.a (mon.0) 834 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-03-07T03:11:49.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[31149]: audit 2024-03-07T03:11:49.005475+0000 mon.a (mon.0) 835 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:49 smithi148 bash[31149]: audit 2024-03-07T03:11:49.046382+0000 mon.b (mon.1) 110 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:49.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:49 smithi195 bash[23221]: audit 2024-03-07T03:11:49.001253+0000 mon.b (mon.1) 107 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:49 smithi195 bash[23221]: audit 2024-03-07T03:11:49.001815+0000 mon.b (mon.1) 108 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-03-07T03:11:49.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:49 smithi195 bash[23221]: audit 2024-03-07T03:11:49.002289+0000 mon.b (mon.1) 109 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:49 smithi195 bash[23221]: audit 2024-03-07T03:11:49.004737+0000 mon.a (mon.0) 833 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:49 smithi195 bash[23221]: audit 2024-03-07T03:11:49.005023+0000 mon.a (mon.0) 834 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2024-03-07T03:11:49.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:49 smithi195 bash[23221]: audit 2024-03-07T03:11:49.005475+0000 mon.a (mon.0) 835 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2024-03-07T03:11:49.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:49 smithi195 bash[23221]: audit 2024-03-07T03:11:49.046382+0000 mon.b (mon.1) 110 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:11:50.696 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:11:50.697 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":72,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:11:50.471234+0000","last_up_change":"2024-03-07T03:07:53.161981+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:09:29.853395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:11:16.277829+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:11:17.121168+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6803","nonce":1285694746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6805","nonce":1285694746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6809","nonce":1285694746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6807","nonce":1285694746}]},"public_addr":"172.21.15.148:6803/1285694746","cluster_addr":"172.21.15.148:6805/1285694746","heartbeat_back_addr":"172.21.15.148:6809/1285694746","heartbeat_front_addr":"172.21.15.148:6807/1285694746","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6811","nonce":3763364013}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6813","nonce":3763364013}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6817","nonce":3763364013}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6815","nonce":3763364013}]},"public_addr":"172.21.15.148:6811/3763364013","cluster_addr":"172.21.15.148:6813/3763364013","heartbeat_back_addr":"172.21.15.148:6817/3763364013","heartbeat_front_addr":"172.21.15.148:6815/3763364013","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6819","nonce":916957575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6821","nonce":916957575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6825","nonce":916957575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6823","nonce":916957575}]},"public_addr":"172.21.15.148:6819/916957575","cluster_addr":"172.21.15.148:6821/916957575","heartbeat_back_addr":"172.21.15.148:6825/916957575","heartbeat_front_addr":"172.21.15.148:6823/916957575","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6827","nonce":2267330328}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6829","nonce":2267330328}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6833","nonce":2267330328}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6831","nonce":2267330328}]},"public_addr":"172.21.15.148:6827/2267330328","cluster_addr":"172.21.15.148:6829/2267330328","heartbeat_back_addr":"172.21.15.148:6833/2267330328","heartbeat_front_addr":"172.21.15.148:6831/2267330328","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6801","nonce":1363408518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6803","nonce":1363408518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6807","nonce":1363408518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6805","nonce":1363408518}]},"public_addr":"172.21.15.195:6801/1363408518","cluster_addr":"172.21.15.195:6803/1363408518","heartbeat_back_addr":"172.21.15.195:6807/1363408518","heartbeat_front_addr":"172.21.15.195:6805/1363408518","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:6834/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2258249874":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6835/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3220133019":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/1901736382":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4143961239":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/480238330":"2024-03-08T03:08:48.946763+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[21362]: cluster 2024-03-07T03:11:49.022992+0000 mgr.y (mgr.24287) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[21362]: audit 2024-03-07T03:11:49.479893+0000 mon.a (mon.0) 836 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[21362]: audit 2024-03-07T03:11:49.480121+0000 mon.a (mon.0) 837 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[21362]: audit 2024-03-07T03:11:49.480730+0000 mon.a (mon.0) 838 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[21362]: cluster 2024-03-07T03:11:49.480934+0000 mon.a (mon.0) 839 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[31149]: cluster 2024-03-07T03:11:49.022992+0000 mgr.y (mgr.24287) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[31149]: audit 2024-03-07T03:11:49.479893+0000 mon.a (mon.0) 836 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[31149]: audit 2024-03-07T03:11:49.480121+0000 mon.a (mon.0) 837 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[31149]: audit 2024-03-07T03:11:49.480730+0000 mon.a (mon.0) 838 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-07T03:11:50.710 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:50 smithi148 bash[31149]: cluster 2024-03-07T03:11:49.480934+0000 mon.a (mon.0) 839 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-03-07T03:11:50.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:50 smithi195 bash[23221]: cluster 2024-03-07T03:11:49.022992+0000 mgr.y (mgr.24287) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:50.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:50 smithi195 bash[23221]: audit 2024-03-07T03:11:49.479893+0000 mon.a (mon.0) 836 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.4", "id": [1, 5]}]': finished 2024-03-07T03:11:50.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:50 smithi195 bash[23221]: audit 2024-03-07T03:11:49.480121+0000 mon.a (mon.0) 837 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2024-03-07T03:11:50.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:50 smithi195 bash[23221]: audit 2024-03-07T03:11:49.480730+0000 mon.a (mon.0) 838 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2024-03-07T03:11:50.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:50 smithi195 bash[23221]: cluster 2024-03-07T03:11:49.480934+0000 mon.a (mon.0) 839 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-03-07T03:11:51.236 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-07T03:11:51.237 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:11:51.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:51 smithi148 bash[21362]: cluster 2024-03-07T03:11:50.481053+0000 mon.a (mon.0) 840 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-03-07T03:11:51.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:51 smithi148 bash[21362]: audit 2024-03-07T03:11:50.705287+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.148:0/2446258517' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:11:51.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:51 smithi148 bash[31149]: cluster 2024-03-07T03:11:50.481053+0000 mon.a (mon.0) 840 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-03-07T03:11:51.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:51 smithi148 bash[31149]: audit 2024-03-07T03:11:50.705287+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.148:0/2446258517' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:11:51.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:51 smithi195 bash[23221]: cluster 2024-03-07T03:11:50.481053+0000 mon.a (mon.0) 840 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-03-07T03:11:51.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:51 smithi195 bash[23221]: audit 2024-03-07T03:11:50.705287+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.148:0/2446258517' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:11:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:52 smithi148 bash[21362]: cluster 2024-03-07T03:11:51.023595+0000 mgr.y (mgr.24287) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:52.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:52 smithi148 bash[31149]: cluster 2024-03-07T03:11:51.023595+0000 mgr.y (mgr.24287) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:52.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:52 smithi195 bash[23221]: cluster 2024-03-07T03:11:51.023595+0000 mgr.y (mgr.24287) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:54 smithi148 bash[21362]: cluster 2024-03-07T03:11:53.024252+0000 mgr.y (mgr.24287) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:54 smithi148 bash[31149]: cluster 2024-03-07T03:11:53.024252+0000 mgr.y (mgr.24287) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:54.792 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:11:54 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:11:54] "GET /metrics HTTP/1.1" 200 33910 "" "Prometheus/2.43.0" 2024-03-07T03:11:54.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:54 smithi195 bash[23221]: cluster 2024-03-07T03:11:53.024252+0000 mgr.y (mgr.24287) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:55.976 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:11:56.749 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:56 smithi148 bash[21362]: cluster 2024-03-07T03:11:55.025264+0000 mgr.y (mgr.24287) 133 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:56.750 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:56 smithi148 bash[31149]: cluster 2024-03-07T03:11:55.025264+0000 mgr.y (mgr.24287) 133 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:56.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:56 smithi195 bash[23221]: cluster 2024-03-07T03:11:55.025264+0000 mgr.y (mgr.24287) 133 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:58.482 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:11:58.482 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":72,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:11:50.471234+0000","last_up_change":"2024-03-07T03:07:53.161981+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:09:29.853395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:11:16.277829+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:11:17.121168+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6803","nonce":1285694746}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6805","nonce":1285694746}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6809","nonce":1285694746}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":1285694746},{"type":"v1","addr":"172.21.15.148:6807","nonce":1285694746}]},"public_addr":"172.21.15.148:6803/1285694746","cluster_addr":"172.21.15.148:6805/1285694746","heartbeat_back_addr":"172.21.15.148:6809/1285694746","heartbeat_front_addr":"172.21.15.148:6807/1285694746","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6811","nonce":3763364013}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6813","nonce":3763364013}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6817","nonce":3763364013}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":3763364013},{"type":"v1","addr":"172.21.15.148:6815","nonce":3763364013}]},"public_addr":"172.21.15.148:6811/3763364013","cluster_addr":"172.21.15.148:6813/3763364013","heartbeat_back_addr":"172.21.15.148:6817/3763364013","heartbeat_front_addr":"172.21.15.148:6815/3763364013","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6819","nonce":916957575}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6821","nonce":916957575}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6825","nonce":916957575}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":916957575},{"type":"v1","addr":"172.21.15.148:6823","nonce":916957575}]},"public_addr":"172.21.15.148:6819/916957575","cluster_addr":"172.21.15.148:6821/916957575","heartbeat_back_addr":"172.21.15.148:6825/916957575","heartbeat_front_addr":"172.21.15.148:6823/916957575","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6827","nonce":2267330328}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6829","nonce":2267330328}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6833","nonce":2267330328}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":2267330328},{"type":"v1","addr":"172.21.15.148:6831","nonce":2267330328}]},"public_addr":"172.21.15.148:6827/2267330328","cluster_addr":"172.21.15.148:6829/2267330328","heartbeat_back_addr":"172.21.15.148:6833/2267330328","heartbeat_front_addr":"172.21.15.148:6831/2267330328","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6801","nonce":1363408518}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6803","nonce":1363408518}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6807","nonce":1363408518}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":1363408518},{"type":"v1","addr":"172.21.15.195:6805","nonce":1363408518}]},"public_addr":"172.21.15.195:6801/1363408518","cluster_addr":"172.21.15.195:6803/1363408518","heartbeat_back_addr":"172.21.15.195:6807/1363408518","heartbeat_front_addr":"172.21.15.195:6805/1363408518","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:6834/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2258249874":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6835/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3220133019":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/1901736382":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/4143961239":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/480238330":"2024-03-08T03:08:48.946763+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:11:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:58 smithi148 bash[21362]: cluster 2024-03-07T03:11:57.025945+0000 mgr.y (mgr.24287) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:11:58 smithi148 bash[21362]: audit 2024-03-07T03:11:58.491499+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.148:0/3804172471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:11:58.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:58 smithi148 bash[31149]: cluster 2024-03-07T03:11:57.025945+0000 mgr.y (mgr.24287) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:58.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:11:58 smithi148 bash[31149]: audit 2024-03-07T03:11:58.491499+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.148:0/3804172471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:11:58.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:58 smithi195 bash[23221]: cluster 2024-03-07T03:11:57.025945+0000 mgr.y (mgr.24287) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:11:58.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:11:58 smithi195 bash[23221]: audit 2024-03-07T03:11:58.491499+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.148:0/3804172471' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:11:59.068 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-07T03:11:59.068 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-07T03:11:59.069 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-07T03:11:59.069 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-07T03:11:59.070 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-07T03:11:59.070 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-07T03:11:59.070 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-07T03:11:59.071 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-07T03:12:00.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:00 smithi148 bash[21362]: cluster 2024-03-07T03:11:59.026714+0000 mgr.y (mgr.24287) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:00 smithi148 bash[31149]: cluster 2024-03-07T03:11:59.026714+0000 mgr.y (mgr.24287) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:00.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:00 smithi195 bash[23221]: cluster 2024-03-07T03:11:59.026714+0000 mgr.y (mgr.24287) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:02.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:02 smithi148 bash[21362]: cluster 2024-03-07T03:12:01.027843+0000 mgr.y (mgr.24287) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:02.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:02 smithi148 bash[31149]: cluster 2024-03-07T03:12:01.027843+0000 mgr.y (mgr.24287) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:02.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:02 smithi195 bash[23221]: cluster 2024-03-07T03:12:01.027843+0000 mgr.y (mgr.24287) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:03.897 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:03.897 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:03.897 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:03.898 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:03.901 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:03.901 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:03.902 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:03.902 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:04.791 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:12:04 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:12:04] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.43.0" 2024-03-07T03:12:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:04 smithi148 bash[21362]: cluster 2024-03-07T03:12:03.028478+0000 mgr.y (mgr.24287) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:04.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:04 smithi148 bash[21362]: audit 2024-03-07T03:12:04.047259+0000 mon.b (mon.1) 111 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:04.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:04 smithi148 bash[31149]: cluster 2024-03-07T03:12:03.028478+0000 mgr.y (mgr.24287) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:04.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:04 smithi148 bash[31149]: audit 2024-03-07T03:12:04.047259+0000 mon.b (mon.1) 111 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:04.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:04 smithi195 bash[23221]: cluster 2024-03-07T03:12:03.028478+0000 mgr.y (mgr.24287) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:04.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:04 smithi195 bash[23221]: audit 2024-03-07T03:12:04.047259+0000 mon.b (mon.1) 111 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:06 smithi148 bash[21362]: cluster 2024-03-07T03:12:05.029052+0000 mgr.y (mgr.24287) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:06 smithi148 bash[31149]: cluster 2024-03-07T03:12:05.029052+0000 mgr.y (mgr.24287) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:06.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:06 smithi195 bash[23221]: cluster 2024-03-07T03:12:05.029052+0000 mgr.y (mgr.24287) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:08.632 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:08 smithi148 bash[21362]: cluster 2024-03-07T03:12:07.029772+0000 mgr.y (mgr.24287) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:08.633 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:08 smithi148 bash[31149]: cluster 2024-03-07T03:12:07.029772+0000 mgr.y (mgr.24287) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:08.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:08 smithi195 bash[23221]: cluster 2024-03-07T03:12:07.029772+0000 mgr.y (mgr.24287) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:10 smithi148 bash[21362]: cluster 2024-03-07T03:12:09.030430+0000 mgr.y (mgr.24287) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:10 smithi148 bash[31149]: cluster 2024-03-07T03:12:09.030430+0000 mgr.y (mgr.24287) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:10.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:10 smithi195 bash[23221]: cluster 2024-03-07T03:12:09.030430+0000 mgr.y (mgr.24287) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:12 smithi148 bash[21362]: cluster 2024-03-07T03:12:11.031536+0000 mgr.y (mgr.24287) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:12 smithi148 bash[31149]: cluster 2024-03-07T03:12:11.031536+0000 mgr.y (mgr.24287) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:12.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:12 smithi195 bash[23221]: cluster 2024-03-07T03:12:11.031536+0000 mgr.y (mgr.24287) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:14.633 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:14 smithi148 bash[31149]: cluster 2024-03-07T03:12:13.032247+0000 mgr.y (mgr.24287) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:14.633 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:14 smithi148 bash[21362]: cluster 2024-03-07T03:12:13.032247+0000 mgr.y (mgr.24287) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:14.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:14 smithi195 bash[23221]: cluster 2024-03-07T03:12:13.032247+0000 mgr.y (mgr.24287) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:15.042 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:12:14 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:12:14] "GET /metrics HTTP/1.1" 200 33911 "" "Prometheus/2.43.0" 2024-03-07T03:12:15.047 INFO:teuthology.orchestra.run.smithi148.stdout:77309411432 2024-03-07T03:12:15.047 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-07T03:12:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:16 smithi148 bash[21362]: cluster 2024-03-07T03:12:15.033292+0000 mgr.y (mgr.24287) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:16.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:16 smithi148 bash[31149]: cluster 2024-03-07T03:12:15.033292+0000 mgr.y (mgr.24287) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:16.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:16 smithi195 bash[23221]: cluster 2024-03-07T03:12:15.033292+0000 mgr.y (mgr.24287) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:16.886 INFO:teuthology.orchestra.run.smithi148.stdout:111669149791 2024-03-07T03:12:16.886 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-07T03:12:16.912 INFO:teuthology.orchestra.run.smithi148.stdout:55834574963 2024-03-07T03:12:16.912 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-07T03:12:17.716 INFO:teuthology.orchestra.run.smithi148.stdout:167503724620 2024-03-07T03:12:17.716 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-07T03:12:18.630 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:18 smithi148 bash[21362]: cluster 2024-03-07T03:12:17.033797+0000 mgr.y (mgr.24287) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:18.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:18 smithi195 bash[23221]: cluster 2024-03-07T03:12:17.033797+0000 mgr.y (mgr.24287) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:19.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:18 smithi148 bash[31149]: cluster 2024-03-07T03:12:17.033797+0000 mgr.y (mgr.24287) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:19.089 INFO:teuthology.orchestra.run.smithi148.stdout:34359738493 2024-03-07T03:12:19.090 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-07T03:12:19.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:19 smithi195 bash[23221]: audit 2024-03-07T03:12:19.047724+0000 mon.b (mon.1) 112 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:19.831 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:19 smithi148 bash[21362]: audit 2024-03-07T03:12:19.047724+0000 mon.b (mon.1) 112 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:19.831 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:19 smithi148 bash[31149]: audit 2024-03-07T03:12:19.047724+0000 mon.b (mon.1) 112 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:19.832 INFO:teuthology.orchestra.run.smithi148.stdout:236223201335 2024-03-07T03:12:19.832 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-07T03:12:19.847 INFO:teuthology.orchestra.run.smithi148.stdout:201863462978 2024-03-07T03:12:19.847 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-07T03:12:19.919 INFO:teuthology.orchestra.run.smithi148.stdout:133143986261 2024-03-07T03:12:19.919 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-07T03:12:20.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:20 smithi195 bash[23221]: cluster 2024-03-07T03:12:19.034666+0000 mgr.y (mgr.24287) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:21.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:20 smithi148 bash[21362]: cluster 2024-03-07T03:12:19.034666+0000 mgr.y (mgr.24287) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:21.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:20 smithi148 bash[31149]: cluster 2024-03-07T03:12:19.034666+0000 mgr.y (mgr.24287) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:22.352 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:22.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:22 smithi148 bash[21362]: cluster 2024-03-07T03:12:21.035902+0000 mgr.y (mgr.24287) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:22.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:22 smithi148 bash[31149]: cluster 2024-03-07T03:12:21.035902+0000 mgr.y (mgr.24287) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:22.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:22 smithi195 bash[23221]: cluster 2024-03-07T03:12:21.035902+0000 mgr.y (mgr.24287) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:23.369 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:23.370 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:23.370 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:24.631 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:24 smithi148 bash[31149]: cluster 2024-03-07T03:12:23.036377+0000 mgr.y (mgr.24287) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:24.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:24 smithi148 bash[21362]: cluster 2024-03-07T03:12:23.036377+0000 mgr.y (mgr.24287) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:24.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:24 smithi195 bash[23221]: cluster 2024-03-07T03:12:23.036377+0000 mgr.y (mgr.24287) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:25.040 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:12:24 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:12:24] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2024-03-07T03:12:25.082 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:25.082 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:25.083 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:25.083 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:26.811 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:26 smithi148 bash[31149]: cluster 2024-03-07T03:12:25.037335+0000 mgr.y (mgr.24287) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:26.811 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:26 smithi148 bash[21362]: cluster 2024-03-07T03:12:25.037335+0000 mgr.y (mgr.24287) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:26.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:26 smithi195 bash[23221]: cluster 2024-03-07T03:12:25.037335+0000 mgr.y (mgr.24287) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:27.338 INFO:teuthology.orchestra.run.smithi148.stdout:77309411435 2024-03-07T03:12:27.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:27 smithi148 bash[21362]: audit 2024-03-07T03:12:27.351103+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.148:0/2878769516' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:12:27.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:27 smithi148 bash[31149]: audit 2024-03-07T03:12:27.351103+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.148:0/2878769516' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:12:27.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:27 smithi195 bash[23221]: audit 2024-03-07T03:12:27.351103+0000 mon.a (mon.0) 841 : audit 0 from='client.? 172.21.15.148:0/2878769516' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:12:28.828 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:28 smithi148 bash[21362]: cluster 2024-03-07T03:12:27.037776+0000 mgr.y (mgr.24287) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:28.829 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:28 smithi148 bash[31149]: cluster 2024-03-07T03:12:27.037776+0000 mgr.y (mgr.24287) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:29.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:28 smithi195 bash[23221]: cluster 2024-03-07T03:12:27.037776+0000 mgr.y (mgr.24287) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:30.645 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411432 got 77309411435 for osd.2 2024-03-07T03:12:30.646 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:31.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:30 smithi195 bash[23221]: cluster 2024-03-07T03:12:29.038326+0000 mgr.y (mgr.24287) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:31.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:30 smithi195 bash[23221]: audit 2024-03-07T03:12:29.640853+0000 mon.b (mon.1) 113 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:31.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:30 smithi195 bash[23221]: audit 2024-03-07T03:12:30.219623+0000 mon.b (mon.1) 114 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:12:31.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:30 smithi195 bash[23221]: audit 2024-03-07T03:12:30.221504+0000 mon.b (mon.1) 115 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:12:31.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:30 smithi195 bash[23221]: audit 2024-03-07T03:12:30.231563+0000 mon.a (mon.0) 842 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:12:31.075 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[21362]: cluster 2024-03-07T03:12:29.038326+0000 mgr.y (mgr.24287) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:31.075 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[21362]: audit 2024-03-07T03:12:29.640853+0000 mon.b (mon.1) 113 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:31.075 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[21362]: audit 2024-03-07T03:12:30.219623+0000 mon.b (mon.1) 114 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:12:31.075 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[21362]: audit 2024-03-07T03:12:30.221504+0000 mon.b (mon.1) 115 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:12:31.075 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[21362]: audit 2024-03-07T03:12:30.231563+0000 mon.a (mon.0) 842 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:12:31.075 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[31149]: cluster 2024-03-07T03:12:29.038326+0000 mgr.y (mgr.24287) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:31.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[31149]: audit 2024-03-07T03:12:29.640853+0000 mon.b (mon.1) 113 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:12:31.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[31149]: audit 2024-03-07T03:12:30.219623+0000 mon.b (mon.1) 114 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:12:31.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[31149]: audit 2024-03-07T03:12:30.221504+0000 mon.b (mon.1) 115 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:12:31.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:30 smithi148 bash[31149]: audit 2024-03-07T03:12:30.231563+0000 mon.a (mon.0) 842 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:12:32.866 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:32 smithi148 bash[21362]: cluster 2024-03-07T03:12:31.039499+0000 mgr.y (mgr.24287) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:32.866 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:32 smithi148 bash[31149]: cluster 2024-03-07T03:12:31.039499+0000 mgr.y (mgr.24287) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:33.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:32 smithi195 bash[23221]: cluster 2024-03-07T03:12:31.039499+0000 mgr.y (mgr.24287) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:34.372 INFO:teuthology.orchestra.run.smithi148.stdout:111669149795 2024-03-07T03:12:34.489 INFO:teuthology.orchestra.run.smithi148.stdout:55834574966 2024-03-07T03:12:34.596 INFO:teuthology.orchestra.run.smithi148.stdout:167503724623 2024-03-07T03:12:34.790 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:12:34] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[21362]: cluster 2024-03-07T03:12:33.040102+0000 mgr.y (mgr.24287) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[21362]: audit 2024-03-07T03:12:34.047203+0000 mon.b (mon.1) 116 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[21362]: audit 2024-03-07T03:12:34.384826+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.148:0/96646713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[21362]: audit 2024-03-07T03:12:34.502777+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.148:0/1990434984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[31149]: cluster 2024-03-07T03:12:33.040102+0000 mgr.y (mgr.24287) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[31149]: audit 2024-03-07T03:12:34.047203+0000 mon.b (mon.1) 116 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[31149]: audit 2024-03-07T03:12:34.384826+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.148:0/96646713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:12:34.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:34 smithi148 bash[31149]: audit 2024-03-07T03:12:34.502777+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.148:0/1990434984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:12:35.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:34 smithi195 bash[23221]: cluster 2024-03-07T03:12:33.040102+0000 mgr.y (mgr.24287) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:35.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:34 smithi195 bash[23221]: audit 2024-03-07T03:12:34.047203+0000 mon.b (mon.1) 116 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:35.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:34 smithi195 bash[23221]: audit 2024-03-07T03:12:34.384826+0000 mon.a (mon.0) 843 : audit 0 from='client.? 172.21.15.148:0/96646713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:12:35.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:34 smithi195 bash[23221]: audit 2024-03-07T03:12:34.502777+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.148:0/1990434984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:12:35.344 INFO:teuthology.orchestra.run.smithi148.stdout:236223201338 2024-03-07T03:12:35.615 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:35 smithi148 bash[21362]: audit 2024-03-07T03:12:34.609742+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.148:0/433134034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:12:35.615 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:35 smithi148 bash[21362]: audit 2024-03-07T03:12:35.357559+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.148:0/3532315995' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:12:36.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:35 smithi148 bash[31149]: audit 2024-03-07T03:12:34.609742+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.148:0/433134034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:12:36.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:35 smithi148 bash[31149]: audit 2024-03-07T03:12:35.357559+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.148:0/3532315995' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:12:36.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:35 smithi195 bash[23221]: audit 2024-03-07T03:12:34.609742+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.148:0/433134034' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:12:36.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:35 smithi195 bash[23221]: audit 2024-03-07T03:12:35.357559+0000 mon.a (mon.0) 844 : audit 0 from='client.? 172.21.15.148:0/3532315995' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:12:37.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:36 smithi148 bash[21362]: cluster 2024-03-07T03:12:35.040761+0000 mgr.y (mgr.24287) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:37.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:36 smithi148 bash[31149]: cluster 2024-03-07T03:12:35.040761+0000 mgr.y (mgr.24287) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:37.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:36 smithi195 bash[23221]: cluster 2024-03-07T03:12:35.040761+0000 mgr.y (mgr.24287) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:37.115 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149791 got 111669149795 for osd.3 2024-03-07T03:12:37.115 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:37.444 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574963 got 55834574966 for osd.1 2024-03-07T03:12:37.445 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:37.761 INFO:teuthology.orchestra.run.smithi148.stdout:133143986265 2024-03-07T03:12:38.632 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724620 got 167503724623 for osd.5 2024-03-07T03:12:38.632 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:38.664 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201335 got 236223201338 for osd.7 2024-03-07T03:12:38.664 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:38.852 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:38 smithi148 bash[21362]: cluster 2024-03-07T03:12:37.041363+0000 mgr.y (mgr.24287) 154 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:38.852 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:38 smithi148 bash[21362]: audit 2024-03-07T03:12:37.774593+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.148:0/1063798960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:12:38.852 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:38 smithi148 bash[31149]: cluster 2024-03-07T03:12:37.041363+0000 mgr.y (mgr.24287) 154 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:38.852 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:38 smithi148 bash[31149]: audit 2024-03-07T03:12:37.774593+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.148:0/1063798960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:12:39.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:38 smithi195 bash[23221]: cluster 2024-03-07T03:12:37.041363+0000 mgr.y (mgr.24287) 154 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:39.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:38 smithi195 bash[23221]: audit 2024-03-07T03:12:37.774593+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.148:0/1063798960' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:12:39.218 INFO:teuthology.orchestra.run.smithi148.stdout:201863462982 2024-03-07T03:12:39.646 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986261 got 133143986265 for osd.4 2024-03-07T03:12:39.646 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:39.901 INFO:teuthology.orchestra.run.smithi148.stdout:34359738497 2024-03-07T03:12:39.915 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:39 smithi148 bash[21362]: audit 2024-03-07T03:12:39.231418+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.148:0/4120510597' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:12:39.916 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:39 smithi148 bash[31149]: audit 2024-03-07T03:12:39.231418+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.148:0/4120510597' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:12:40.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:39 smithi195 bash[23221]: audit 2024-03-07T03:12:39.231418+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.148:0/4120510597' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:12:40.309 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462978 got 201863462982 for osd.6 2024-03-07T03:12:40.309 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:40.657 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738493 got 34359738497 for osd.0 2024-03-07T03:12:40.657 DEBUG:teuthology.parallel:result is None 2024-03-07T03:12:40.658 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-07T03:12:40.658 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:12:41.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:40 smithi148 bash[21362]: cluster 2024-03-07T03:12:39.041757+0000 mgr.y (mgr.24287) 155 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:41.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:40 smithi148 bash[21362]: audit 2024-03-07T03:12:39.914099+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.148:0/1278498852' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:12:41.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:40 smithi148 bash[31149]: cluster 2024-03-07T03:12:39.041757+0000 mgr.y (mgr.24287) 155 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:41.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:40 smithi148 bash[31149]: audit 2024-03-07T03:12:39.914099+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.148:0/1278498852' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:12:41.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:40 smithi195 bash[23221]: cluster 2024-03-07T03:12:39.041757+0000 mgr.y (mgr.24287) 155 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:41.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:40 smithi195 bash[23221]: audit 2024-03-07T03:12:39.914099+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.148:0/1278498852' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:12:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:42 smithi148 bash[21362]: cluster 2024-03-07T03:12:41.042763+0000 mgr.y (mgr.24287) 156 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:42 smithi148 bash[31149]: cluster 2024-03-07T03:12:41.042763+0000 mgr.y (mgr.24287) 156 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:43.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:42 smithi195 bash[23221]: cluster 2024-03-07T03:12:41.042763+0000 mgr.y (mgr.24287) 156 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:45.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:44 smithi148 bash[21362]: cluster 2024-03-07T03:12:43.043403+0000 mgr.y (mgr.24287) 157 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:45.041 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:12:44 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:12:44] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2024-03-07T03:12:45.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:44 smithi148 bash[31149]: cluster 2024-03-07T03:12:43.043403+0000 mgr.y (mgr.24287) 157 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:45.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:44 smithi195 bash[23221]: cluster 2024-03-07T03:12:43.043403+0000 mgr.y (mgr.24287) 157 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:45.389 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:46.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:46 smithi148 bash[21362]: cluster 2024-03-07T03:12:45.044469+0000 mgr.y (mgr.24287) 158 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:46.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:46 smithi148 bash[31149]: cluster 2024-03-07T03:12:45.044469+0000 mgr.y (mgr.24287) 158 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:47.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:46 smithi195 bash[23221]: cluster 2024-03-07T03:12:45.044469+0000 mgr.y (mgr.24287) 158 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:48.075 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:12:48.077 INFO:teuthology.orchestra.run.smithi148.stderr:dumped all 2024-03-07T03:12:48.637 INFO:teuthology.orchestra.run.smithi148.stdout:{"pg_ready":true,"pg_map":{"version":134,"stamp":"2024-03-07T03:12:47.044708+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":289,"num_read_kb":386,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223772,"kb_used_data":7608,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749737444,"statfs":{"total":767960285184,"available":767731142656,"internally_reserved":0,"allocated":7790592,"data_stored":4373331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004371"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.478251+0000","last_change":"2024-03-07T03:11:17.762433+0000","last_active":"2024-03-07T03:11:49.478251+0000","last_peered":"2024-03-07T03:11:49.478251+0000","last_clean":"2024-03-07T03:11:49.478251+0000","last_became_active":"2024-03-07T03:11:17.761957+0000","last_became_peered":"2024-03-07T03:11:17.761957+0000","last_unstale":"2024-03-07T03:11:49.478251+0000","last_undegraded":"2024-03-07T03:11:49.478251+0000","last_fullsized":"2024-03-07T03:11:49.478251+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:22:22.662579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868315+0000","last_change":"2024-03-07T03:11:17.752329+0000","last_active":"2024-03-07T03:11:49.868315+0000","last_peered":"2024-03-07T03:11:49.868315+0000","last_clean":"2024-03-07T03:11:49.868315+0000","last_became_active":"2024-03-07T03:11:17.751695+0000","last_became_peered":"2024-03-07T03:11:17.751695+0000","last_unstale":"2024-03-07T03:11:49.868315+0000","last_undegraded":"2024-03-07T03:11:49.868315+0000","last_fullsized":"2024-03-07T03:11:49.868315+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:31:05.696543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479282+0000","last_change":"2024-03-07T03:11:17.844094+0000","last_active":"2024-03-07T03:11:49.479282+0000","last_peered":"2024-03-07T03:11:49.479282+0000","last_clean":"2024-03-07T03:11:49.479282+0000","last_became_active":"2024-03-07T03:11:17.843769+0000","last_became_peered":"2024-03-07T03:11:17.843769+0000","last_unstale":"2024-03-07T03:11:49.479282+0000","last_undegraded":"2024-03-07T03:11:49.479282+0000","last_fullsized":"2024-03-07T03:11:49.479282+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:20:13.146211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477682+0000","last_change":"2024-03-07T03:11:17.733223+0000","last_active":"2024-03-07T03:11:49.477682+0000","last_peered":"2024-03-07T03:11:49.477682+0000","last_clean":"2024-03-07T03:11:49.477682+0000","last_became_active":"2024-03-07T03:11:17.731893+0000","last_became_peered":"2024-03-07T03:11:17.731893+0000","last_unstale":"2024-03-07T03:11:49.477682+0000","last_undegraded":"2024-03-07T03:11:49.477682+0000","last_fullsized":"2024-03-07T03:11:49.477682+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:21:12.888076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480674+0000","last_change":"2024-03-07T03:11:17.763532+0000","last_active":"2024-03-07T03:11:49.480674+0000","last_peered":"2024-03-07T03:11:49.480674+0000","last_clean":"2024-03-07T03:11:49.480674+0000","last_became_active":"2024-03-07T03:11:17.763206+0000","last_became_peered":"2024-03-07T03:11:17.763206+0000","last_unstale":"2024-03-07T03:11:49.480674+0000","last_undegraded":"2024-03-07T03:11:49.480674+0000","last_fullsized":"2024-03-07T03:11:49.480674+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:04:50.514780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480802+0000","last_change":"2024-03-07T03:11:17.728694+0000","last_active":"2024-03-07T03:11:49.480802+0000","last_peered":"2024-03-07T03:11:49.480802+0000","last_clean":"2024-03-07T03:11:49.480802+0000","last_became_active":"2024-03-07T03:11:17.727704+0000","last_became_peered":"2024-03-07T03:11:17.727704+0000","last_unstale":"2024-03-07T03:11:49.480802+0000","last_undegraded":"2024-03-07T03:11:49.480802+0000","last_fullsized":"2024-03-07T03:11:49.480802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:18:04.455845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.481532+0000","last_change":"2024-03-07T03:11:17.763765+0000","last_active":"2024-03-07T03:11:49.481532+0000","last_peered":"2024-03-07T03:11:49.481532+0000","last_clean":"2024-03-07T03:11:49.481532+0000","last_became_active":"2024-03-07T03:11:17.763361+0000","last_became_peered":"2024-03-07T03:11:17.763361+0000","last_unstale":"2024-03-07T03:11:49.481532+0000","last_undegraded":"2024-03-07T03:11:49.481532+0000","last_fullsized":"2024-03-07T03:11:49.481532+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:42:48.387310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480807+0000","last_change":"2024-03-07T03:11:17.726915+0000","last_active":"2024-03-07T03:11:49.480807+0000","last_peered":"2024-03-07T03:11:49.480807+0000","last_clean":"2024-03-07T03:11:49.480807+0000","last_became_active":"2024-03-07T03:11:17.726078+0000","last_became_peered":"2024-03-07T03:11:17.726078+0000","last_unstale":"2024-03-07T03:11:49.480807+0000","last_undegraded":"2024-03-07T03:11:49.480807+0000","last_fullsized":"2024-03-07T03:11:49.480807+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:00:52.554381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477939+0000","last_change":"2024-03-07T03:11:19.028055+0000","last_active":"2024-03-07T03:11:49.477939+0000","last_peered":"2024-03-07T03:11:49.477939+0000","last_clean":"2024-03-07T03:11:49.477939+0000","last_became_active":"2024-03-07T03:11:19.026706+0000","last_became_peered":"2024-03-07T03:11:19.026706+0000","last_unstale":"2024-03-07T03:11:49.477939+0000","last_undegraded":"2024-03-07T03:11:49.477939+0000","last_fullsized":"2024-03-07T03:11:49.477939+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:27:09.803901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868264+0000","last_change":"2024-03-07T03:11:17.730681+0000","last_active":"2024-03-07T03:11:49.868264+0000","last_peered":"2024-03-07T03:11:49.868264+0000","last_clean":"2024-03-07T03:11:49.868264+0000","last_became_active":"2024-03-07T03:11:17.729900+0000","last_became_peered":"2024-03-07T03:11:17.729900+0000","last_unstale":"2024-03-07T03:11:49.868264+0000","last_undegraded":"2024-03-07T03:11:49.868264+0000","last_fullsized":"2024-03-07T03:11:49.868264+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:43:00.084654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482328+0000","last_change":"2024-03-07T03:11:17.753267+0000","last_active":"2024-03-07T03:11:49.482328+0000","last_peered":"2024-03-07T03:11:49.482328+0000","last_clean":"2024-03-07T03:11:49.482328+0000","last_became_active":"2024-03-07T03:11:17.752977+0000","last_became_peered":"2024-03-07T03:11:17.752977+0000","last_unstale":"2024-03-07T03:11:49.482328+0000","last_undegraded":"2024-03-07T03:11:49.482328+0000","last_fullsized":"2024-03-07T03:11:49.482328+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:00:24.404134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868429+0000","last_change":"2024-03-07T03:11:17.735077+0000","last_active":"2024-03-07T03:11:49.868429+0000","last_peered":"2024-03-07T03:11:49.868429+0000","last_clean":"2024-03-07T03:11:49.868429+0000","last_became_active":"2024-03-07T03:11:17.731073+0000","last_became_peered":"2024-03-07T03:11:17.731073+0000","last_unstale":"2024-03-07T03:11:49.868429+0000","last_undegraded":"2024-03-07T03:11:49.868429+0000","last_fullsized":"2024-03-07T03:11:49.868429+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:08:07.371789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480953+0000","last_change":"2024-03-07T03:09:31.339012+0000","last_active":"2024-03-07T03:11:49.480953+0000","last_peered":"2024-03-07T03:11:49.480953+0000","last_clean":"2024-03-07T03:11:49.480953+0000","last_became_active":"2024-03-07T03:09:31.338487+0000","last_became_peered":"2024-03-07T03:09:31.338487+0000","last_unstale":"2024-03-07T03:11:49.480953+0000","last_undegraded":"2024-03-07T03:11:49.480953+0000","last_fullsized":"2024-03-07T03:11:49.480953+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:48:57.153468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.474725+0000","last_change":"2024-03-07T03:11:17.850632+0000","last_active":"2024-03-07T03:11:49.474725+0000","last_peered":"2024-03-07T03:11:49.474725+0000","last_clean":"2024-03-07T03:11:49.474725+0000","last_became_active":"2024-03-07T03:11:17.850392+0000","last_became_peered":"2024-03-07T03:11:17.850392+0000","last_unstale":"2024-03-07T03:11:49.474725+0000","last_undegraded":"2024-03-07T03:11:49.474725+0000","last_fullsized":"2024-03-07T03:11:49.474725+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:41:12.881887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487226+0000","last_change":"2024-03-07T03:09:31.492237+0000","last_active":"2024-03-07T03:11:49.487226+0000","last_peered":"2024-03-07T03:11:49.487226+0000","last_clean":"2024-03-07T03:11:49.487226+0000","last_became_active":"2024-03-07T03:09:31.490659+0000","last_became_peered":"2024-03-07T03:09:31.490659+0000","last_unstale":"2024-03-07T03:11:49.487226+0000","last_undegraded":"2024-03-07T03:11:49.487226+0000","last_fullsized":"2024-03-07T03:11:49.487226+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:58:14.548445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480366+0000","last_change":"2024-03-07T03:11:17.744707+0000","last_active":"2024-03-07T03:11:49.480366+0000","last_peered":"2024-03-07T03:11:49.480366+0000","last_clean":"2024-03-07T03:11:49.480366+0000","last_became_active":"2024-03-07T03:11:17.744274+0000","last_became_peered":"2024-03-07T03:11:17.744274+0000","last_unstale":"2024-03-07T03:11:49.480366+0000","last_undegraded":"2024-03-07T03:11:49.480366+0000","last_fullsized":"2024-03-07T03:11:49.480366+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:40:12.704059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869230+0000","last_change":"2024-03-07T03:11:19.034716+0000","last_active":"2024-03-07T03:11:49.869230+0000","last_peered":"2024-03-07T03:11:49.869230+0000","last_clean":"2024-03-07T03:11:49.869230+0000","last_became_active":"2024-03-07T03:11:19.032902+0000","last_became_peered":"2024-03-07T03:11:19.032902+0000","last_unstale":"2024-03-07T03:11:49.869230+0000","last_undegraded":"2024-03-07T03:11:49.869230+0000","last_fullsized":"2024-03-07T03:11:49.869230+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:38:08.075495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487222+0000","last_change":"2024-03-07T03:11:17.737252+0000","last_active":"2024-03-07T03:11:49.487222+0000","last_peered":"2024-03-07T03:11:49.487222+0000","last_clean":"2024-03-07T03:11:49.487222+0000","last_became_active":"2024-03-07T03:11:17.736246+0000","last_became_peered":"2024-03-07T03:11:17.736246+0000","last_unstale":"2024-03-07T03:11:49.487222+0000","last_undegraded":"2024-03-07T03:11:49.487222+0000","last_fullsized":"2024-03-07T03:11:49.487222+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:49:42.107373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473523+0000","last_change":"2024-03-07T03:09:31.490148+0000","last_active":"2024-03-07T03:11:49.473523+0000","last_peered":"2024-03-07T03:11:49.473523+0000","last_clean":"2024-03-07T03:11:49.473523+0000","last_became_active":"2024-03-07T03:09:31.489708+0000","last_became_peered":"2024-03-07T03:09:31.489708+0000","last_unstale":"2024-03-07T03:11:49.473523+0000","last_undegraded":"2024-03-07T03:11:49.473523+0000","last_fullsized":"2024-03-07T03:11:49.473523+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:21:06.820876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869757+0000","last_change":"2024-03-07T03:11:19.029566+0000","last_active":"2024-03-07T03:11:49.869757+0000","last_peered":"2024-03-07T03:11:49.869757+0000","last_clean":"2024-03-07T03:11:49.869757+0000","last_became_active":"2024-03-07T03:11:19.028550+0000","last_became_peered":"2024-03-07T03:11:19.028550+0000","last_unstale":"2024-03-07T03:11:49.869757+0000","last_undegraded":"2024-03-07T03:11:49.869757+0000","last_fullsized":"2024-03-07T03:11:49.869757+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:35:52.637946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.867810+0000","last_change":"2024-03-07T03:11:17.741573+0000","last_active":"2024-03-07T03:11:49.867810+0000","last_peered":"2024-03-07T03:11:49.867810+0000","last_clean":"2024-03-07T03:11:49.867810+0000","last_became_active":"2024-03-07T03:11:17.736518+0000","last_became_peered":"2024-03-07T03:11:17.736518+0000","last_unstale":"2024-03-07T03:11:49.867810+0000","last_undegraded":"2024-03-07T03:11:49.867810+0000","last_fullsized":"2024-03-07T03:11:49.867810+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:47:03.005123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.472243+0000","last_change":"2024-03-07T03:09:31.490070+0000","last_active":"2024-03-07T03:11:49.472243+0000","last_peered":"2024-03-07T03:11:49.472243+0000","last_clean":"2024-03-07T03:11:49.472243+0000","last_became_active":"2024-03-07T03:09:31.489632+0000","last_became_peered":"2024-03-07T03:09:31.489632+0000","last_unstale":"2024-03-07T03:11:49.472243+0000","last_undegraded":"2024-03-07T03:11:49.472243+0000","last_fullsized":"2024-03-07T03:11:49.472243+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:08:00.120410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482066+0000","last_change":"2024-03-07T03:11:17.849560+0000","last_active":"2024-03-07T03:11:49.482066+0000","last_peered":"2024-03-07T03:11:49.482066+0000","last_clean":"2024-03-07T03:11:49.482066+0000","last_became_active":"2024-03-07T03:11:17.848681+0000","last_became_peered":"2024-03-07T03:11:17.848681+0000","last_unstale":"2024-03-07T03:11:49.482066+0000","last_undegraded":"2024-03-07T03:11:49.482066+0000","last_fullsized":"2024-03-07T03:11:49.482066+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:29:42.167986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475798+0000","last_change":"2024-03-07T03:11:17.727584+0000","last_active":"2024-03-07T03:11:49.475798+0000","last_peered":"2024-03-07T03:11:49.475798+0000","last_clean":"2024-03-07T03:11:49.475798+0000","last_became_active":"2024-03-07T03:11:17.726901+0000","last_became_peered":"2024-03-07T03:11:17.726901+0000","last_unstale":"2024-03-07T03:11:49.475798+0000","last_undegraded":"2024-03-07T03:11:49.475798+0000","last_fullsized":"2024-03-07T03:11:49.475798+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:12:30.904702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":269,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:50.656673+0000","last_change":"2024-03-07T03:07:57.209434+0000","last_active":"2024-03-07T03:11:50.656673+0000","last_peered":"2024-03-07T03:11:50.656673+0000","last_clean":"2024-03-07T03:11:50.656673+0000","last_became_active":"2024-03-07T03:07:57.208461+0000","last_became_peered":"2024-03-07T03:07:57.208461+0000","last_unstale":"2024-03-07T03:11:50.656673+0000","last_undegraded":"2024-03-07T03:11:50.656673+0000","last_fullsized":"2024-03-07T03:11:50.656673+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_clean_scrub_stamp":"2024-03-07T03:03:44.169894+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:55:26.915133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479911+0000","last_change":"2024-03-07T03:09:31.338124+0000","last_active":"2024-03-07T03:11:49.479911+0000","last_peered":"2024-03-07T03:11:49.479911+0000","last_clean":"2024-03-07T03:11:49.479911+0000","last_became_active":"2024-03-07T03:09:31.337531+0000","last_became_peered":"2024-03-07T03:09:31.337531+0000","last_unstale":"2024-03-07T03:11:49.479911+0000","last_undegraded":"2024-03-07T03:11:49.479911+0000","last_fullsized":"2024-03-07T03:11:49.479911+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:39.200015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487610+0000","last_change":"2024-03-07T03:11:17.768359+0000","last_active":"2024-03-07T03:11:49.487610+0000","last_peered":"2024-03-07T03:11:49.487610+0000","last_clean":"2024-03-07T03:11:49.487610+0000","last_became_active":"2024-03-07T03:11:17.767774+0000","last_became_peered":"2024-03-07T03:11:17.767774+0000","last_unstale":"2024-03-07T03:11:49.487610+0000","last_undegraded":"2024-03-07T03:11:49.487610+0000","last_fullsized":"2024-03-07T03:11:49.487610+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:20:11.902070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485106+0000","last_change":"2024-03-07T03:11:17.746565+0000","last_active":"2024-03-07T03:11:49.485106+0000","last_peered":"2024-03-07T03:11:49.485106+0000","last_clean":"2024-03-07T03:11:49.485106+0000","last_became_active":"2024-03-07T03:11:17.744126+0000","last_became_peered":"2024-03-07T03:11:17.744126+0000","last_unstale":"2024-03-07T03:11:49.485106+0000","last_undegraded":"2024-03-07T03:11:49.485106+0000","last_fullsized":"2024-03-07T03:11:49.485106+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:14:03.441173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477100+0000","last_change":"2024-03-07T03:09:31.492946+0000","last_active":"2024-03-07T03:11:49.477100+0000","last_peered":"2024-03-07T03:11:49.477100+0000","last_clean":"2024-03-07T03:11:49.477100+0000","last_became_active":"2024-03-07T03:09:31.491949+0000","last_became_peered":"2024-03-07T03:09:31.491949+0000","last_unstale":"2024-03-07T03:11:49.477100+0000","last_undegraded":"2024-03-07T03:11:49.477100+0000","last_fullsized":"2024-03-07T03:11:49.477100+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:42:06.966194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:51.152125+0000","last_change":"2024-03-07T03:11:51.152125+0000","last_active":"2024-03-07T03:11:51.152125+0000","last_peered":"2024-03-07T03:11:51.152125+0000","last_clean":"2024-03-07T03:11:51.152125+0000","last_became_active":"2024-03-07T03:11:51.150853+0000","last_became_peered":"2024-03-07T03:11:51.150853+0000","last_unstale":"2024-03-07T03:11:51.152125+0000","last_undegraded":"2024-03-07T03:11:51.152125+0000","last_fullsized":"2024-03-07T03:11:51.152125+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:20:44.149315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.481289+0000","last_change":"2024-03-07T03:11:17.734091+0000","last_active":"2024-03-07T03:11:49.481289+0000","last_peered":"2024-03-07T03:11:49.481289+0000","last_clean":"2024-03-07T03:11:49.481289+0000","last_became_active":"2024-03-07T03:11:17.733811+0000","last_became_peered":"2024-03-07T03:11:17.733811+0000","last_unstale":"2024-03-07T03:11:49.481289+0000","last_undegraded":"2024-03-07T03:11:49.481289+0000","last_fullsized":"2024-03-07T03:11:49.481289+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:35:23.691468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479579+0000","last_change":"2024-03-07T03:09:31.340350+0000","last_active":"2024-03-07T03:11:49.479579+0000","last_peered":"2024-03-07T03:11:49.479579+0000","last_clean":"2024-03-07T03:11:49.479579+0000","last_became_active":"2024-03-07T03:09:31.339524+0000","last_became_peered":"2024-03-07T03:09:31.339524+0000","last_unstale":"2024-03-07T03:11:49.479579+0000","last_undegraded":"2024-03-07T03:11:49.479579+0000","last_fullsized":"2024-03-07T03:11:49.479579+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:14:32.989838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485520+0000","last_change":"2024-03-07T03:11:17.763549+0000","last_active":"2024-03-07T03:11:49.485520+0000","last_peered":"2024-03-07T03:11:49.485520+0000","last_clean":"2024-03-07T03:11:49.485520+0000","last_became_active":"2024-03-07T03:11:17.762902+0000","last_became_peered":"2024-03-07T03:11:17.762902+0000","last_unstale":"2024-03-07T03:11:49.485520+0000","last_undegraded":"2024-03-07T03:11:49.485520+0000","last_fullsized":"2024-03-07T03:11:49.485520+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:24:59.291166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480521+0000","last_change":"2024-03-07T03:11:17.736543+0000","last_active":"2024-03-07T03:11:49.480521+0000","last_peered":"2024-03-07T03:11:49.480521+0000","last_clean":"2024-03-07T03:11:49.480521+0000","last_became_active":"2024-03-07T03:11:17.735167+0000","last_became_peered":"2024-03-07T03:11:17.735167+0000","last_unstale":"2024-03-07T03:11:49.480521+0000","last_undegraded":"2024-03-07T03:11:49.480521+0000","last_fullsized":"2024-03-07T03:11:49.480521+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:06:39.189368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.486988+0000","last_change":"2024-03-07T03:09:31.493118+0000","last_active":"2024-03-07T03:11:49.486988+0000","last_peered":"2024-03-07T03:11:49.486988+0000","last_clean":"2024-03-07T03:11:49.486988+0000","last_became_active":"2024-03-07T03:09:31.491952+0000","last_became_peered":"2024-03-07T03:09:31.491952+0000","last_unstale":"2024-03-07T03:11:49.486988+0000","last_undegraded":"2024-03-07T03:11:49.486988+0000","last_fullsized":"2024-03-07T03:11:49.486988+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:52:26.178237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487689+0000","last_change":"2024-03-07T03:11:17.762401+0000","last_active":"2024-03-07T03:11:49.487689+0000","last_peered":"2024-03-07T03:11:49.487689+0000","last_clean":"2024-03-07T03:11:49.487689+0000","last_became_active":"2024-03-07T03:11:17.761846+0000","last_became_peered":"2024-03-07T03:11:17.761846+0000","last_unstale":"2024-03-07T03:11:49.487689+0000","last_undegraded":"2024-03-07T03:11:49.487689+0000","last_fullsized":"2024-03-07T03:11:49.487689+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:54:19.206136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.474837+0000","last_change":"2024-03-07T03:11:17.722629+0000","last_active":"2024-03-07T03:11:49.474837+0000","last_peered":"2024-03-07T03:11:49.474837+0000","last_clean":"2024-03-07T03:11:49.474837+0000","last_became_active":"2024-03-07T03:11:17.721320+0000","last_became_peered":"2024-03-07T03:11:17.721320+0000","last_unstale":"2024-03-07T03:11:49.474837+0000","last_undegraded":"2024-03-07T03:11:49.474837+0000","last_fullsized":"2024-03-07T03:11:49.474837+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:29:11.930358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.483483+0000","last_change":"2024-03-07T03:11:17.763117+0000","last_active":"2024-03-07T03:11:49.483483+0000","last_peered":"2024-03-07T03:11:49.483483+0000","last_clean":"2024-03-07T03:11:49.483483+0000","last_became_active":"2024-03-07T03:11:17.761698+0000","last_became_peered":"2024-03-07T03:11:17.761698+0000","last_unstale":"2024-03-07T03:11:49.483483+0000","last_undegraded":"2024-03-07T03:11:49.483483+0000","last_fullsized":"2024-03-07T03:11:49.483483+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:34:22.206821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482449+0000","last_change":"2024-03-07T03:11:17.736365+0000","last_active":"2024-03-07T03:11:49.482449+0000","last_peered":"2024-03-07T03:11:49.482449+0000","last_clean":"2024-03-07T03:11:49.482449+0000","last_became_active":"2024-03-07T03:11:17.728478+0000","last_became_peered":"2024-03-07T03:11:17.728478+0000","last_unstale":"2024-03-07T03:11:49.482449+0000","last_undegraded":"2024-03-07T03:11:49.482449+0000","last_fullsized":"2024-03-07T03:11:49.482449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:24:39.629084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:51.152721+0000","last_change":"2024-03-07T03:11:51.152721+0000","last_active":"2024-03-07T03:11:51.152721+0000","last_peered":"2024-03-07T03:11:51.152721+0000","last_clean":"2024-03-07T03:11:51.152721+0000","last_became_active":"2024-03-07T03:11:51.151718+0000","last_became_peered":"2024-03-07T03:11:51.151718+0000","last_unstale":"2024-03-07T03:11:51.152721+0000","last_undegraded":"2024-03-07T03:11:51.152721+0000","last_fullsized":"2024-03-07T03:11:51.152721+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:16:19.214983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480726+0000","last_change":"2024-03-07T03:11:17.727286+0000","last_active":"2024-03-07T03:11:49.480726+0000","last_peered":"2024-03-07T03:11:49.480726+0000","last_clean":"2024-03-07T03:11:49.480726+0000","last_became_active":"2024-03-07T03:11:17.726975+0000","last_became_peered":"2024-03-07T03:11:17.726975+0000","last_unstale":"2024-03-07T03:11:49.480726+0000","last_undegraded":"2024-03-07T03:11:49.480726+0000","last_fullsized":"2024-03-07T03:11:49.480726+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:46:19.775858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480492+0000","last_change":"2024-03-07T03:11:17.762607+0000","last_active":"2024-03-07T03:11:49.480492+0000","last_peered":"2024-03-07T03:11:49.480492+0000","last_clean":"2024-03-07T03:11:49.480492+0000","last_became_active":"2024-03-07T03:11:17.761393+0000","last_became_peered":"2024-03-07T03:11:17.761393+0000","last_unstale":"2024-03-07T03:11:49.480492+0000","last_undegraded":"2024-03-07T03:11:49.480492+0000","last_fullsized":"2024-03-07T03:11:49.480492+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:38:34.500610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475557+0000","last_change":"2024-03-07T03:11:17.745662+0000","last_active":"2024-03-07T03:11:49.475557+0000","last_peered":"2024-03-07T03:11:49.475557+0000","last_clean":"2024-03-07T03:11:49.475557+0000","last_became_active":"2024-03-07T03:11:17.745370+0000","last_became_peered":"2024-03-07T03:11:17.745370+0000","last_unstale":"2024-03-07T03:11:49.475557+0000","last_undegraded":"2024-03-07T03:11:49.475557+0000","last_fullsized":"2024-03-07T03:11:49.475557+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:58:02.187340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869070+0000","last_change":"2024-03-07T03:11:19.035152+0000","last_active":"2024-03-07T03:11:49.869070+0000","last_peered":"2024-03-07T03:11:49.869070+0000","last_clean":"2024-03-07T03:11:49.869070+0000","last_became_active":"2024-03-07T03:11:19.033898+0000","last_became_peered":"2024-03-07T03:11:19.033898+0000","last_unstale":"2024-03-07T03:11:49.869070+0000","last_undegraded":"2024-03-07T03:11:49.869070+0000","last_fullsized":"2024-03-07T03:11:49.869070+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:39:57.234259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480635+0000","last_change":"2024-03-07T03:11:17.722831+0000","last_active":"2024-03-07T03:11:49.480635+0000","last_peered":"2024-03-07T03:11:49.480635+0000","last_clean":"2024-03-07T03:11:49.480635+0000","last_became_active":"2024-03-07T03:11:17.720768+0000","last_became_peered":"2024-03-07T03:11:17.720768+0000","last_unstale":"2024-03-07T03:11:49.480635+0000","last_undegraded":"2024-03-07T03:11:49.480635+0000","last_fullsized":"2024-03-07T03:11:49.480635+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:11:28.934497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:51.153475+0000","last_change":"2024-03-07T03:11:51.153475+0000","last_active":"2024-03-07T03:11:51.153475+0000","last_peered":"2024-03-07T03:11:51.153475+0000","last_clean":"2024-03-07T03:11:51.153475+0000","last_became_active":"2024-03-07T03:11:51.152067+0000","last_became_peered":"2024-03-07T03:11:51.152067+0000","last_unstale":"2024-03-07T03:11:51.153475+0000","last_undegraded":"2024-03-07T03:11:51.153475+0000","last_fullsized":"2024-03-07T03:11:51.153475+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:28:48.025654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473642+0000","last_change":"2024-03-07T03:11:17.738764+0000","last_active":"2024-03-07T03:11:49.473642+0000","last_peered":"2024-03-07T03:11:49.473642+0000","last_clean":"2024-03-07T03:11:49.473642+0000","last_became_active":"2024-03-07T03:11:17.738151+0000","last_became_peered":"2024-03-07T03:11:17.738151+0000","last_unstale":"2024-03-07T03:11:49.473642+0000","last_undegraded":"2024-03-07T03:11:49.473642+0000","last_fullsized":"2024-03-07T03:11:49.473642+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:37:22.289110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482158+0000","last_change":"2024-03-07T03:11:17.762914+0000","last_active":"2024-03-07T03:11:49.482158+0000","last_peered":"2024-03-07T03:11:49.482158+0000","last_clean":"2024-03-07T03:11:49.482158+0000","last_became_active":"2024-03-07T03:11:17.762339+0000","last_became_peered":"2024-03-07T03:11:17.762339+0000","last_unstale":"2024-03-07T03:11:49.482158+0000","last_undegraded":"2024-03-07T03:11:49.482158+0000","last_fullsized":"2024-03-07T03:11:49.482158+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:01:57.515149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473161+0000","last_change":"2024-03-07T03:11:17.737593+0000","last_active":"2024-03-07T03:11:49.473161+0000","last_peered":"2024-03-07T03:11:49.473161+0000","last_clean":"2024-03-07T03:11:49.473161+0000","last_became_active":"2024-03-07T03:11:17.732115+0000","last_became_peered":"2024-03-07T03:11:17.732115+0000","last_unstale":"2024-03-07T03:11:49.473161+0000","last_undegraded":"2024-03-07T03:11:49.473161+0000","last_fullsized":"2024-03-07T03:11:49.473161+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:31:45.710698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479259+0000","last_change":"2024-03-07T03:11:17.846593+0000","last_active":"2024-03-07T03:11:49.479259+0000","last_peered":"2024-03-07T03:11:49.479259+0000","last_clean":"2024-03-07T03:11:49.479259+0000","last_became_active":"2024-03-07T03:11:17.845734+0000","last_became_peered":"2024-03-07T03:11:17.845734+0000","last_unstale":"2024-03-07T03:11:49.479259+0000","last_undegraded":"2024-03-07T03:11:49.479259+0000","last_fullsized":"2024-03-07T03:11:49.479259+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:18:40.161672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473071+0000","last_change":"2024-03-07T03:11:17.735133+0000","last_active":"2024-03-07T03:11:49.473071+0000","last_peered":"2024-03-07T03:11:49.473071+0000","last_clean":"2024-03-07T03:11:49.473071+0000","last_became_active":"2024-03-07T03:11:17.734029+0000","last_became_peered":"2024-03-07T03:11:17.734029+0000","last_unstale":"2024-03-07T03:11:49.473071+0000","last_undegraded":"2024-03-07T03:11:49.473071+0000","last_fullsized":"2024-03-07T03:11:49.473071+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:51:27.606846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475621+0000","last_change":"2024-03-07T03:11:17.847015+0000","last_active":"2024-03-07T03:11:49.475621+0000","last_peered":"2024-03-07T03:11:49.475621+0000","last_clean":"2024-03-07T03:11:49.475621+0000","last_became_active":"2024-03-07T03:11:17.846740+0000","last_became_peered":"2024-03-07T03:11:17.846740+0000","last_unstale":"2024-03-07T03:11:49.475621+0000","last_undegraded":"2024-03-07T03:11:49.475621+0000","last_fullsized":"2024-03-07T03:11:49.475621+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:51:17.128266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":32,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477033+0000","last_change":"2024-03-07T03:11:17.750812+0000","last_active":"2024-03-07T03:11:49.477033+0000","last_peered":"2024-03-07T03:11:49.477033+0000","last_clean":"2024-03-07T03:11:49.477033+0000","last_became_active":"2024-03-07T03:11:17.750471+0000","last_became_peered":"2024-03-07T03:11:17.750471+0000","last_unstale":"2024-03-07T03:11:49.477033+0000","last_undegraded":"2024-03-07T03:11:49.477033+0000","last_fullsized":"2024-03-07T03:11:49.477033+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:58:02.992427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869340+0000","last_change":"2024-03-07T03:11:19.031805+0000","last_active":"2024-03-07T03:11:49.869340+0000","last_peered":"2024-03-07T03:11:49.869340+0000","last_clean":"2024-03-07T03:11:49.869340+0000","last_became_active":"2024-03-07T03:11:19.031219+0000","last_became_peered":"2024-03-07T03:11:19.031219+0000","last_unstale":"2024-03-07T03:11:49.869340+0000","last_undegraded":"2024-03-07T03:11:49.869340+0000","last_fullsized":"2024-03-07T03:11:49.869340+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:25:58.130337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.472079+0000","last_change":"2024-03-07T03:11:17.733218+0000","last_active":"2024-03-07T03:11:49.472079+0000","last_peered":"2024-03-07T03:11:49.472079+0000","last_clean":"2024-03-07T03:11:49.472079+0000","last_became_active":"2024-03-07T03:11:17.732035+0000","last_became_peered":"2024-03-07T03:11:17.732035+0000","last_unstale":"2024-03-07T03:11:49.472079+0000","last_undegraded":"2024-03-07T03:11:49.472079+0000","last_fullsized":"2024-03-07T03:11:49.472079+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:29:32.508319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480279+0000","last_change":"2024-03-07T03:11:17.849679+0000","last_active":"2024-03-07T03:11:49.480279+0000","last_peered":"2024-03-07T03:11:49.480279+0000","last_clean":"2024-03-07T03:11:49.480279+0000","last_became_active":"2024-03-07T03:11:17.848732+0000","last_became_peered":"2024-03-07T03:11:17.848732+0000","last_unstale":"2024-03-07T03:11:49.480279+0000","last_undegraded":"2024-03-07T03:11:49.480279+0000","last_fullsized":"2024-03-07T03:11:49.480279+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:32:32.048244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868028+0000","last_change":"2024-03-07T03:11:17.729899+0000","last_active":"2024-03-07T03:11:49.868028+0000","last_peered":"2024-03-07T03:11:49.868028+0000","last_clean":"2024-03-07T03:11:49.868028+0000","last_became_active":"2024-03-07T03:11:17.729108+0000","last_became_peered":"2024-03-07T03:11:17.729108+0000","last_unstale":"2024-03-07T03:11:49.868028+0000","last_undegraded":"2024-03-07T03:11:49.868028+0000","last_fullsized":"2024-03-07T03:11:49.868028+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:31:13.744056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477933+0000","last_change":"2024-03-07T03:11:17.846954+0000","last_active":"2024-03-07T03:11:49.477933+0000","last_peered":"2024-03-07T03:11:49.477933+0000","last_clean":"2024-03-07T03:11:49.477933+0000","last_became_active":"2024-03-07T03:11:17.846652+0000","last_became_peered":"2024-03-07T03:11:17.846652+0000","last_unstale":"2024-03-07T03:11:49.477933+0000","last_undegraded":"2024-03-07T03:11:49.477933+0000","last_fullsized":"2024-03-07T03:11:49.477933+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:18:30.656475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.472893+0000","last_change":"2024-03-07T03:11:17.730834+0000","last_active":"2024-03-07T03:11:49.472893+0000","last_peered":"2024-03-07T03:11:49.472893+0000","last_clean":"2024-03-07T03:11:49.472893+0000","last_became_active":"2024-03-07T03:11:17.730541+0000","last_became_peered":"2024-03-07T03:11:17.730541+0000","last_unstale":"2024-03-07T03:11:49.472893+0000","last_undegraded":"2024-03-07T03:11:49.472893+0000","last_fullsized":"2024-03-07T03:11:49.472893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:41:51.335682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480845+0000","last_change":"2024-03-07T03:11:17.762938+0000","last_active":"2024-03-07T03:11:49.480845+0000","last_peered":"2024-03-07T03:11:49.480845+0000","last_clean":"2024-03-07T03:11:49.480845+0000","last_became_active":"2024-03-07T03:11:17.762293+0000","last_became_peered":"2024-03-07T03:11:17.762293+0000","last_unstale":"2024-03-07T03:11:49.480845+0000","last_undegraded":"2024-03-07T03:11:49.480845+0000","last_fullsized":"2024-03-07T03:11:49.480845+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:47:14.683735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":29,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482881+0000","last_change":"2024-03-07T03:11:17.721344+0000","last_active":"2024-03-07T03:11:49.482881+0000","last_peered":"2024-03-07T03:11:49.482881+0000","last_clean":"2024-03-07T03:11:49.482881+0000","last_became_active":"2024-03-07T03:11:17.720857+0000","last_became_peered":"2024-03-07T03:11:17.720857+0000","last_unstale":"2024-03-07T03:11:49.482881+0000","last_undegraded":"2024-03-07T03:11:49.482881+0000","last_fullsized":"2024-03-07T03:11:49.482881+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:14.558664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485350+0000","last_change":"2024-03-07T03:11:17.844288+0000","last_active":"2024-03-07T03:11:49.485350+0000","last_peered":"2024-03-07T03:11:49.485350+0000","last_clean":"2024-03-07T03:11:49.485350+0000","last_became_active":"2024-03-07T03:11:17.843897+0000","last_became_peered":"2024-03-07T03:11:17.843897+0000","last_unstale":"2024-03-07T03:11:49.485350+0000","last_undegraded":"2024-03-07T03:11:49.485350+0000","last_fullsized":"2024-03-07T03:11:49.485350+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:48:48.959645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473409+0000","last_change":"2024-03-07T03:11:17.754277+0000","last_active":"2024-03-07T03:11:49.473409+0000","last_peered":"2024-03-07T03:11:49.473409+0000","last_clean":"2024-03-07T03:11:49.473409+0000","last_became_active":"2024-03-07T03:11:17.753979+0000","last_became_peered":"2024-03-07T03:11:17.753979+0000","last_unstale":"2024-03-07T03:11:49.473409+0000","last_undegraded":"2024-03-07T03:11:49.473409+0000","last_fullsized":"2024-03-07T03:11:49.473409+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:00:02.786312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479516+0000","last_change":"2024-03-07T03:11:17.850841+0000","last_active":"2024-03-07T03:11:49.479516+0000","last_peered":"2024-03-07T03:11:49.479516+0000","last_clean":"2024-03-07T03:11:49.479516+0000","last_became_active":"2024-03-07T03:11:17.850452+0000","last_became_peered":"2024-03-07T03:11:17.850452+0000","last_unstale":"2024-03-07T03:11:49.479516+0000","last_undegraded":"2024-03-07T03:11:49.479516+0000","last_fullsized":"2024-03-07T03:11:49.479516+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:11:24.249898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.481058+0000","last_change":"2024-03-07T03:11:17.731035+0000","last_active":"2024-03-07T03:11:49.481058+0000","last_peered":"2024-03-07T03:11:49.481058+0000","last_clean":"2024-03-07T03:11:49.481058+0000","last_became_active":"2024-03-07T03:11:17.730092+0000","last_became_peered":"2024-03-07T03:11:17.730092+0000","last_unstale":"2024-03-07T03:11:49.481058+0000","last_undegraded":"2024-03-07T03:11:49.481058+0000","last_fullsized":"2024-03-07T03:11:49.481058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:20:50.529165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475051+0000","last_change":"2024-03-07T03:11:17.761532+0000","last_active":"2024-03-07T03:11:49.475051+0000","last_peered":"2024-03-07T03:11:49.475051+0000","last_clean":"2024-03-07T03:11:49.475051+0000","last_became_active":"2024-03-07T03:11:17.761092+0000","last_became_peered":"2024-03-07T03:11:17.761092+0000","last_unstale":"2024-03-07T03:11:49.475051+0000","last_undegraded":"2024-03-07T03:11:49.475051+0000","last_fullsized":"2024-03-07T03:11:49.475051+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:51:19.912304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868469+0000","last_change":"2024-03-07T03:11:17.750987+0000","last_active":"2024-03-07T03:11:49.868469+0000","last_peered":"2024-03-07T03:11:49.868469+0000","last_clean":"2024-03-07T03:11:49.868469+0000","last_became_active":"2024-03-07T03:11:17.750258+0000","last_became_peered":"2024-03-07T03:11:17.750258+0000","last_unstale":"2024-03-07T03:11:49.868469+0000","last_undegraded":"2024-03-07T03:11:49.868469+0000","last_fullsized":"2024-03-07T03:11:49.868469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:40:37.611600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.478345+0000","last_change":"2024-03-07T03:11:17.758817+0000","last_active":"2024-03-07T03:11:49.478345+0000","last_peered":"2024-03-07T03:11:49.478345+0000","last_clean":"2024-03-07T03:11:49.478345+0000","last_became_active":"2024-03-07T03:11:17.758445+0000","last_became_peered":"2024-03-07T03:11:17.758445+0000","last_unstale":"2024-03-07T03:11:49.478345+0000","last_undegraded":"2024-03-07T03:11:49.478345+0000","last_fullsized":"2024-03-07T03:11:49.478345+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:09.182998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475096+0000","last_change":"2024-03-07T03:11:17.725981+0000","last_active":"2024-03-07T03:11:49.475096+0000","last_peered":"2024-03-07T03:11:49.475096+0000","last_clean":"2024-03-07T03:11:49.475096+0000","last_became_active":"2024-03-07T03:11:17.722925+0000","last_became_peered":"2024-03-07T03:11:17.722925+0000","last_unstale":"2024-03-07T03:11:49.475096+0000","last_undegraded":"2024-03-07T03:11:49.475096+0000","last_fullsized":"2024-03-07T03:11:49.475096+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:19:53.431520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869554+0000","last_change":"2024-03-07T03:11:19.031057+0000","last_active":"2024-03-07T03:11:49.869554+0000","last_peered":"2024-03-07T03:11:49.869554+0000","last_clean":"2024-03-07T03:11:49.869554+0000","last_became_active":"2024-03-07T03:11:19.030633+0000","last_became_peered":"2024-03-07T03:11:19.030633+0000","last_unstale":"2024-03-07T03:11:49.869554+0000","last_undegraded":"2024-03-07T03:11:49.869554+0000","last_fullsized":"2024-03-07T03:11:49.869554+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:44:01.959381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485076+0000","last_change":"2024-03-07T03:11:17.740434+0000","last_active":"2024-03-07T03:11:49.485076+0000","last_peered":"2024-03-07T03:11:49.485076+0000","last_clean":"2024-03-07T03:11:49.485076+0000","last_became_active":"2024-03-07T03:11:17.739612+0000","last_became_peered":"2024-03-07T03:11:17.739612+0000","last_unstale":"2024-03-07T03:11:49.485076+0000","last_undegraded":"2024-03-07T03:11:49.485076+0000","last_fullsized":"2024-03-07T03:11:49.485076+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:55:39.231992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.478841+0000","last_change":"2024-03-07T03:11:17.762095+0000","last_active":"2024-03-07T03:11:49.478841+0000","last_peered":"2024-03-07T03:11:49.478841+0000","last_clean":"2024-03-07T03:11:49.478841+0000","last_became_active":"2024-03-07T03:11:17.761802+0000","last_became_peered":"2024-03-07T03:11:17.761802+0000","last_unstale":"2024-03-07T03:11:49.478841+0000","last_undegraded":"2024-03-07T03:11:49.478841+0000","last_fullsized":"2024-03-07T03:11:49.478841+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:40.710327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480480+0000","last_change":"2024-03-07T03:11:17.733057+0000","last_active":"2024-03-07T03:11:49.480480+0000","last_peered":"2024-03-07T03:11:49.480480+0000","last_clean":"2024-03-07T03:11:49.480480+0000","last_became_active":"2024-03-07T03:11:17.732709+0000","last_became_peered":"2024-03-07T03:11:17.732709+0000","last_unstale":"2024-03-07T03:11:49.480480+0000","last_undegraded":"2024-03-07T03:11:49.480480+0000","last_fullsized":"2024-03-07T03:11:49.480480+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:14:18.485549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":77,"num_read_kb":80,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201341,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28352,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716800,"statfs":{"total":95995035648,"available":95966003200,"internally_reserved":0,"allocated":1359872,"data_stored":916402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.92100000000000004,"15min":0.89500000000000002},"min":{"1min":0.57099999999999995,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.641,"5min":1.641,"15min":1.641},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.96399999999999997,"15min":0.96399999999999997},"min":{"1min":0.61099999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.2849999999999999,"5min":2.1909999999999998,"15min":2.1909999999999998},"last":0.81299999999999994}]},{"osd":1,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.006,"15min":0.98899999999999999},"min":{"1min":0.72099999999999997,"5min":0.627,"15min":0.627},"max":{"1min":1.544,"5min":2.274,"15min":2.274},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.94299999999999995,"15min":0.96799999999999997},"min":{"1min":0.47599999999999998,"5min":0.439,"15min":0.439},"max":{"1min":1.125,"5min":1.536,"15min":1.536},"last":0.78200000000000003}]},{"osd":2,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.0029999999999999,"15min":1.079},"min":{"1min":0.63600000000000001,"5min":0.502,"15min":0.502},"max":{"1min":1.7290000000000001,"5min":4.0270000000000001,"15min":4.0270000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.103,"5min":1.04,"15min":1.04},"min":{"1min":0.63800000000000001,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.458,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":1.1200000000000001}]},{"osd":3,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0589999999999999,"15min":1.075},"min":{"1min":0.76200000000000001,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.383,"5min":2.0710000000000002,"15min":2.0710000000000002},"last":1.165},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.0069999999999999,"15min":1.02},"min":{"1min":0.65200000000000002,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.143,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":0.84699999999999998}]},{"osd":4,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.98599999999999999,"15min":0.96799999999999997},"min":{"1min":0.70899999999999996,"5min":0.501,"15min":0.501},"max":{"1min":1.5860000000000001,"5min":1.831,"15min":1.831},"last":1.206},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.99399999999999999,"15min":0.98199999999999998},"min":{"1min":0.52000000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.6819999999999999,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":0.87}]},{"osd":5,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0409999999999999,"15min":1.006},"min":{"1min":0.68100000000000005,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.5009999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.012,"15min":0.99099999999999999},"min":{"1min":0.59499999999999997,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1970000000000001,"5min":2.1419999999999999,"15min":2.1419999999999999},"last":0.89500000000000002}]},{"osd":6,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.0860000000000001,"15min":1.083},"min":{"1min":0.53400000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.474,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.093,"5min":1.0780000000000001,"15min":1.0449999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":0.99299999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462984,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.90800000000000003,"15min":0.85199999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2430000000000001,"5min":1.409,"15min":1.409},"last":0.879},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.90000000000000002,"15min":0.92600000000000005},"min":{"1min":0.53200000000000003,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.2629999999999999,"5min":1.363,"15min":1.363},"last":0.95399999999999996}]},{"osd":1,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.88200000000000001,"15min":0.91200000000000003},"min":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.1579999999999999,"5min":1.258,"15min":1.258},"last":1.0149999999999999},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.94999999999999996,"15min":0.93300000000000005},"min":{"1min":0.51600000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.196,"5min":1.448,"15min":1.448},"last":0.68400000000000005}]},{"osd":2,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.95699999999999996,"15min":0.93999999999999995},"min":{"1min":0.68600000000000005,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.1970000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":0.99099999999999999},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.55500000000000005,"5min":0.37,"15min":0.37},"max":{"1min":1.1890000000000001,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":1.167}]},{"osd":3,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.93999999999999995,"15min":0.94799999999999995},"min":{"1min":0.53400000000000003,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2110000000000001,"5min":1.3,"15min":1.3},"last":1.0920000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.014,"15min":0.998},"min":{"1min":0.66100000000000003,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.28,"5min":1.536,"15min":1.536},"last":0.93500000000000005}]},{"osd":4,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.90700000000000003,"15min":0.871},"min":{"1min":0.45700000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2749999999999999,"5min":1.488,"15min":1.488},"last":0.84799999999999998},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93200000000000005,"15min":0.91000000000000003},"min":{"1min":0.34899999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.282,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.90100000000000002}]},{"osd":5,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.004,"15min":0.94699999999999995},"min":{"1min":0.42699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.3129999999999999,"5min":1.8759999999999999,"15min":1.8759999999999999},"last":1.0489999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.002,"15min":0.996},"min":{"1min":0.39800000000000002,"5min":0.375,"15min":0.375},"max":{"1min":1.2090000000000001,"5min":1.661,"15min":1.661},"last":0.80300000000000005}]},{"osd":7,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.0649999999999999,"15min":1.0289999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2829999999999999,"5min":1.994,"15min":1.994},"last":0.78300000000000003},{"interface":"front","average":{"1min":1.095,"5min":1.117,"15min":1.081},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.3240000000000001,"5min":1.8220000000000001,"15min":1.8220000000000001},"last":1.1220000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574969,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27744,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717408,"statfs":{"total":95995035648,"available":95966625792,"internally_reserved":0,"allocated":741376,"data_stored":324585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:10 2024","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":0.79800000000000004,"15min":0.68200000000000005},"min":{"1min":0.374,"5min":0.29599999999999999,"15min":0.246},"max":{"1min":8.9090000000000007,"5min":8.9090000000000007,"15min":8.9090000000000007},"last":0.622},{"interface":"front","average":{"1min":3.23,"5min":1.3100000000000001,"15min":0.84699999999999998},"min":{"1min":0.40100000000000002,"5min":0.39800000000000002,"15min":0.32100000000000001},"max":{"1min":45.719999999999999,"5min":45.719999999999999,"15min":45.719999999999999},"last":0.75600000000000001}]},{"osd":2,"last update":"Thu Mar 7 03:11:52 2024","interfaces":[{"interface":"back","average":{"1min":1.198,"5min":0.84999999999999998,"15min":0.70499999999999996},"min":{"1min":0.498,"5min":0.33800000000000002,"15min":0.28599999999999998},"max":{"1min":7.3170000000000002,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.872,"5min":0.78100000000000003,"15min":0.67000000000000004},"min":{"1min":0.46300000000000002,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":1.7090000000000001,"5min":1.7090000000000001,"15min":1.7090000000000001},"last":0.83799999999999997}]},{"osd":3,"last update":"Thu Mar 7 03:11:52 2024","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":0.995,"15min":0.85499999999999998},"min":{"1min":0.53800000000000003,"5min":0.372,"15min":0.372},"max":{"1min":16.350000000000001,"5min":16.350000000000001,"15min":16.350000000000001},"last":0.88500000000000001},{"interface":"front","average":{"1min":1.3660000000000001,"5min":1.01,"15min":0.90100000000000002},"min":{"1min":0.55600000000000005,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":8.7319999999999993,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":1.175}]},{"osd":4,"last update":"Thu Mar 7 03:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.1100000000000001,"15min":0.96999999999999997},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.2330000000000001,"5min":14.393000000000001,"15min":14.393000000000001},"last":1.1020000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.123,"15min":1.022},"min":{"1min":0.624,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2450000000000001,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":1.131}]},{"osd":5,"last update":"Thu Mar 7 03:12:24 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.1559999999999999,"15min":1.026},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.411,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":1.254},{"interface":"front","average":{"1min":1.046,"5min":1.1100000000000001,"15min":1.04},"min":{"1min":0.64300000000000002,"5min":0.624,"15min":0.624},"max":{"1min":1.54,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":0.95099999999999996}]},{"osd":6,"last update":"Thu Mar 7 03:12:10 2024","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":1.248,"15min":1.137},"min":{"1min":0.63300000000000001,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":16.262,"5min":16.262,"15min":16.262},"last":1.014},{"interface":"front","average":{"1min":1.542,"5min":1.22,"15min":1.1419999999999999},"min":{"1min":0.69999999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":8.7910000000000004,"5min":8.7910000000000004,"15min":8.7910000000000004},"last":1.0509999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:12:01 2024","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.0980000000000001,"15min":1.0840000000000001},"min":{"1min":0.81100000000000005,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":1.1950000000000001},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.264,"15min":1.218},"min":{"1min":0.67700000000000005,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":1.2190000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738499,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.70799999999999996,"15min":0.60599999999999998},"min":{"1min":0.40999999999999998,"5min":0.253,"15min":0.246},"max":{"1min":0.98299999999999998,"5min":1.494,"15min":1.494},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.74099999999999999,"15min":0.64800000000000002},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.23200000000000001},"max":{"1min":1.0720000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.80200000000000005}]},{"osd":2,"last update":"Thu Mar 7 03:12:03 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.84099999999999997,"15min":0.72599999999999998},"min":{"1min":0.43099999999999999,"5min":0.35599999999999998,"15min":0.29099999999999998},"max":{"1min":1.337,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.1879999999999999},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76500000000000001,"15min":0.70499999999999996},"min":{"1min":0.48099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.032,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.77400000000000002}]},{"osd":3,"last update":"Thu Mar 7 03:11:46 2024","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":0.90900000000000003,"15min":0.81699999999999995},"min":{"1min":0.60599999999999998,"5min":0.376,"15min":0.376},"max":{"1min":1.208,"5min":1.788,"15min":1.788},"last":0.92300000000000004},{"interface":"front","average":{"1min":0.89200000000000002,"5min":0.85999999999999999,"15min":0.81100000000000005},"min":{"1min":0.60199999999999998,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.1839999999999999,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":0.68600000000000005}]},{"osd":4,"last update":"Thu Mar 7 03:12:35 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.93400000000000005,"15min":0.878},"min":{"1min":0.626,"5min":0.60399999999999998,"15min":0.59199999999999997},"max":{"1min":1.137,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.88500000000000001},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.085,"15min":0.98999999999999999},"min":{"1min":0.75600000000000001,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.405,"5min":1.46,"15min":1.46},"last":1.0029999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:12:29 2024","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.0960000000000001,"15min":1.042},"min":{"1min":0.77800000000000002,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.486,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.147},{"interface":"front","average":{"1min":1.123,"5min":1.087,"15min":1.0169999999999999},"min":{"1min":0.66900000000000004,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.4159999999999999,"5min":1.651,"15min":1.651},"last":0.85199999999999998}]},{"osd":6,"last update":"Thu Mar 7 03:12:13 2024","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.1200000000000001,"15min":1.1319999999999999},"min":{"1min":0.64600000000000002,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.4830000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":1.1000000000000001},{"interface":"front","average":{"1min":1.141,"5min":1.125,"15min":1.1120000000000001},"min":{"1min":0.70999999999999996,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.514,"5min":1.7070000000000001,"15min":1.7070000000000001},"last":0.96199999999999997}]},{"osd":7,"last update":"Thu Mar 7 03:12:03 2024","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.0660000000000001,"15min":1.087},"min":{"1min":0.68200000000000005,"5min":0.625,"15min":0.625},"max":{"1min":1.4750000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.0780000000000001},{"interface":"front","average":{"1min":1.228,"5min":1.208,"15min":1.238},"min":{"1min":0.71799999999999997,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.5620000000000001,"5min":1.73,"15min":1.73},"last":1.2549999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411439,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":749568,"data_stored":325201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.70199999999999996,"15min":0.63900000000000001},"min":{"1min":0.437,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.1180000000000001,"5min":1.1459999999999999,"15min":1.2929999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.75800000000000001,"15min":0.65900000000000003},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":0.42399999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.75800000000000001,"15min":0.68500000000000005},"min":{"1min":0.36399999999999999,"5min":0.34200000000000003,"15min":0.33200000000000002},"max":{"1min":1.167,"5min":1.218,"15min":1.218},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.82499999999999996,"15min":0.72799999999999998},"min":{"1min":0.46100000000000002,"5min":0.373,"15min":0.312},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.502}]},{"osd":3,"last update":"Thu Mar 7 03:11:46 2024","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.81699999999999995,"15min":0.80100000000000005},"min":{"1min":0.53600000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.1930000000000001,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.879,"15min":0.84499999999999997},"min":{"1min":0.47599999999999998,"5min":0.438,"15min":0.438},"max":{"1min":1.2490000000000001,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":0.47399999999999998}]},{"osd":4,"last update":"Thu Mar 7 03:12:35 2024","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0109999999999999,"15min":0.99099999999999999},"min":{"1min":0.70199999999999996,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.383,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.91000000000000003,"15min":0.91800000000000004},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.147,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.88}]},{"osd":5,"last update":"Thu Mar 7 03:12:17 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.03,"15min":1.0309999999999999},"min":{"1min":0.66900000000000004,"5min":0.628,"15min":0.628},"max":{"1min":1.325,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.93799999999999994,"15min":0.96599999999999997},"min":{"1min":0.60299999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.2290000000000001,"5min":1.25,"15min":1.25},"last":0.81399999999999995}]},{"osd":6,"last update":"Thu Mar 7 03:12:12 2024","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.056,"15min":1.1080000000000001},"min":{"1min":0.59399999999999997,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.353,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.109,"15min":1.139},"min":{"1min":0.64800000000000002,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.5189999999999999,"5min":1.5700000000000001,"15min":1.5700000000000001},"last":0.68500000000000005}]},{"osd":7,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.0369999999999999,"15min":1.0449999999999999},"min":{"1min":0.82399999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.387,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":0.85099999999999998},{"interface":"front","average":{"1min":1.179,"5min":1.123,"15min":1.139},"min":{"1min":0.73399999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.361,"5min":1.611,"15min":1.611},"last":0.67300000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149798,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27744,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717408,"statfs":{"total":95995035648,"available":95966625792,"internally_reserved":0,"allocated":745472,"data_stored":324675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:11:44 2024","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":0.67800000000000005,"15min":0.67400000000000004},"min":{"1min":0.34499999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.98399999999999999,"5min":1.0580000000000001,"15min":1.6419999999999999},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.71599999999999997,"5min":0.748,"15min":0.72699999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.075,"5min":1.321,"15min":1.321},"last":0.70999999999999996}]},{"osd":1,"last update":"Thu Mar 7 03:11:44 2024","interfaces":[{"interface":"back","average":{"1min":0.71199999999999997,"5min":0.76900000000000002,"15min":0.79400000000000004},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.2150000000000001,"5min":1.2709999999999999,"15min":1.4550000000000001},"last":0.998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.73699999999999999,"15min":0.73799999999999999},"min":{"1min":0.32500000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.127,"5min":1.127,"15min":1.127},"last":0.872}]},{"osd":2,"last update":"Thu Mar 7 03:11:44 2024","interfaces":[{"interface":"back","average":{"1min":0.754,"5min":0.81699999999999995,"15min":0.81299999999999994},"min":{"1min":0.29899999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.0469999999999999,"5min":1.198,"15min":1.5109999999999999},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.73299999999999998,"5min":0.76000000000000001,"15min":0.78100000000000003},"min":{"1min":0.38200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.92300000000000004}]},{"osd":4,"last update":"Thu Mar 7 03:12:28 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.96999999999999997,"15min":0.93500000000000005},"min":{"1min":0.71599999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.292,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.02,"5min":1.012,"15min":0.97799999999999998},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.302}]},{"osd":5,"last update":"Thu Mar 7 03:12:28 2024","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.92500000000000004,"15min":0.95299999999999996},"min":{"1min":0.71499999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.224,"5min":1.2490000000000001,"15min":1.268},"last":1.1539999999999999},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0409999999999999,"15min":1.0569999999999999},"min":{"1min":0.67000000000000004,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.5940000000000001,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":1.125}]},{"osd":6,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.046,"15min":1.0960000000000001},"min":{"1min":0.74099999999999999,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":1.4750000000000001},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.98999999999999999,"15min":1.034},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"max":{"1min":1.3520000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.038}]},{"osd":7,"last update":"Thu Mar 7 03:12:02 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.121,"15min":1.1850000000000001},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.5189999999999999,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.282},{"interface":"front","average":{"1min":1.077,"5min":1.139,"15min":1.2050000000000001},"min":{"1min":0.626,"5min":0.626,"15min":0.626},"max":{"1min":1.419,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.417}]}]},{"osd":4,"up_from":31,"seq":133143986267,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":326022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.88700000000000001,"15min":0.81299999999999994},"min":{"1min":0.64800000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":1.087},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.91100000000000003,"15min":0.84699999999999998},"min":{"1min":0.70799999999999996,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":0.94799999999999995}]},{"osd":1,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.91700000000000004,"15min":0.86699999999999999},"min":{"1min":0.627,"5min":0.497,"15min":0.497},"max":{"1min":1.206,"5min":1.343,"15min":1.343},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.93500000000000005,"15min":0.86499999999999999},"min":{"1min":0.64900000000000002,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":1.236}]},{"osd":2,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":0.96199999999999997,"15min":0.90800000000000003},"min":{"1min":0.64600000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":2.024,"5min":2.024,"15min":2.024},"last":0.98999999999999999},{"interface":"front","average":{"1min":1.0640000000000001,"5min":0.99199999999999999,"15min":0.89900000000000002},"min":{"1min":0.72899999999999998,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":1.3069999999999999}]},{"osd":3,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.96499999999999997,"15min":0.873},"min":{"1min":0.77300000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.851,"5min":1.851,"15min":1.851},"last":1.1619999999999999},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0089999999999999,"15min":0.93200000000000005},"min":{"1min":0.70599999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.0269999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.92200000000000004,"15min":0.78300000000000003},"min":{"1min":0.69699999999999995,"5min":0.41199999999999998,"15min":0.371},"max":{"1min":1.3440000000000001,"5min":1.369,"15min":1.369},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.91900000000000004,"15min":0.82699999999999996},"min":{"1min":0.64500000000000002,"5min":0.48399999999999999,"15min":0.34999999999999998},"max":{"1min":1.127,"5min":1.552,"15min":1.552},"last":1.272}]},{"osd":6,"last update":"Thu Mar 7 03:12:11 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.0069999999999999,"15min":0.92000000000000004},"min":{"1min":0.68300000000000005,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.218,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.0469999999999999,"15min":0.92900000000000005},"min":{"1min":0.73199999999999998,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.4450000000000001,"5min":1.754,"15min":1.754},"last":1.355}]},{"osd":7,"last update":"Thu Mar 7 03:11:56 2024","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.0529999999999999,"15min":1.0169999999999999},"min":{"1min":0.59299999999999997,"5min":0.436,"15min":0.436},"max":{"1min":2.2189999999999999,"5min":2.2189999999999999,"15min":2.2189999999999999},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.127,"5min":1.149,"15min":1.119},"min":{"1min":0.60999999999999999,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":2.0760000000000001,"5min":2.0760000000000001,"15min":2.0760000000000001},"last":1.198}]}]},{"osd":5,"up_from":39,"seq":167503724626,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":745472,"data_stored":324906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.90200000000000002,"15min":0.89900000000000002},"min":{"1min":0.60299999999999998,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.236,"5min":1.597,"15min":1.597},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.93400000000000005,"15min":0.87},"min":{"1min":0.68999999999999995,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.2749999999999999,"5min":1.3,"15min":1.3},"last":1}]},{"osd":1,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.95799999999999996,"15min":0.90700000000000003},"min":{"1min":0.67800000000000005,"5min":0.629,"15min":0.51700000000000002},"max":{"1min":1.2609999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.032},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.92800000000000005,"15min":0.89800000000000002},"min":{"1min":0.65200000000000002,"5min":0.501,"15min":0.501},"max":{"1min":1.23,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.1299999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.92500000000000004,"15min":0.88400000000000001},"min":{"1min":0.64700000000000002,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4259999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.052},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97499999999999998,"15min":0.93000000000000005},"min":{"1min":0.69799999999999995,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.1719999999999999,"5min":1.407,"15min":1.407},"last":0.79400000000000004}]},{"osd":3,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":0.96499999999999997,"15min":0.93100000000000005},"min":{"1min":0.80600000000000005,"5min":0.497,"15min":0.497},"max":{"1min":1.546,"5min":1.546,"15min":1.546},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.071,"5min":1.0249999999999999,"15min":0.99399999999999999},"min":{"1min":0.77900000000000003,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.337,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":0.872}]},{"osd":4,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.90500000000000003,"15min":0.78100000000000003},"min":{"1min":0.69699999999999995,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.2370000000000001,"5min":1.571,"15min":1.571},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.085,"5min":0.95799999999999996,"15min":0.84699999999999998},"min":{"1min":0.81899999999999995,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.6459999999999999,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":0.91500000000000004}]},{"osd":6,"last update":"Thu Mar 7 03:12:15 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.99299999999999999,"15min":0.879},"min":{"1min":0.74299999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.2509999999999999,"5min":1.911,"15min":1.911},"last":1.079},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.0089999999999999,"15min":0.89900000000000002},"min":{"1min":0.78800000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":1.1120000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:12:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.0629999999999999,"15min":1.036},"min":{"1min":0.53800000000000003,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3720000000000001,"5min":1.7,"15min":1.7},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.093,"5min":1.109,"15min":1.0860000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.3280000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":1.218}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:12:48.641 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:12:49.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:48 smithi148 bash[21362]: cluster 2024-03-07T03:12:47.045044+0000 mgr.y (mgr.24287) 159 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:49.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:48 smithi148 bash[31149]: cluster 2024-03-07T03:12:47.045044+0000 mgr.y (mgr.24287) 159 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:49.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:48 smithi195 bash[23221]: cluster 2024-03-07T03:12:47.045044+0000 mgr.y (mgr.24287) 159 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:50.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:49 smithi148 bash[21362]: audit 2024-03-07T03:12:48.077404+0000 mgr.y (mgr.24287) 160 : audit 0 from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:12:50.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:49 smithi148 bash[21362]: audit 2024-03-07T03:12:49.048168+0000 mon.b (mon.1) 117 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:50.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:49 smithi148 bash[31149]: audit 2024-03-07T03:12:48.077404+0000 mgr.y (mgr.24287) 160 : audit 0 from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:12:50.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:49 smithi148 bash[31149]: audit 2024-03-07T03:12:49.048168+0000 mon.b (mon.1) 117 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:50.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:49 smithi195 bash[23221]: audit 2024-03-07T03:12:48.077404+0000 mgr.y (mgr.24287) 160 : audit 0 from='client.14712 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:12:50.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:49 smithi195 bash[23221]: audit 2024-03-07T03:12:49.048168+0000 mon.b (mon.1) 117 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:12:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:50 smithi148 bash[21362]: cluster 2024-03-07T03:12:49.045778+0000 mgr.y (mgr.24287) 161 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:51.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:50 smithi148 bash[31149]: cluster 2024-03-07T03:12:49.045778+0000 mgr.y (mgr.24287) 161 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:51.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:50 smithi195 bash[23221]: cluster 2024-03-07T03:12:49.045778+0000 mgr.y (mgr.24287) 161 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:53.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:52 smithi148 bash[21362]: cluster 2024-03-07T03:12:51.046851+0000 mgr.y (mgr.24287) 162 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:53.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:52 smithi148 bash[31149]: cluster 2024-03-07T03:12:51.046851+0000 mgr.y (mgr.24287) 162 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:53.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:52 smithi195 bash[23221]: cluster 2024-03-07T03:12:51.046851+0000 mgr.y (mgr.24287) 162 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:53.381 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:12:54.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:54 smithi148 bash[21362]: cluster 2024-03-07T03:12:53.047521+0000 mgr.y (mgr.24287) 163 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:54.790 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:12:54 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:12:54] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2024-03-07T03:12:54.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:54 smithi148 bash[31149]: cluster 2024-03-07T03:12:53.047521+0000 mgr.y (mgr.24287) 163 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:55.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:54 smithi195 bash[23221]: cluster 2024-03-07T03:12:53.047521+0000 mgr.y (mgr.24287) 163 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:56.092 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:12:56.093 INFO:teuthology.orchestra.run.smithi148.stderr:dumped all 2024-03-07T03:12:56.688 INFO:teuthology.orchestra.run.smithi148.stdout:{"pg_ready":true,"pg_map":{"version":138,"stamp":"2024-03-07T03:12:55.047813+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":289,"num_read_kb":386,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223772,"kb_used_data":7608,"kb_used_omap":13,"kb_used_meta":216050,"kb_avail":749737444,"statfs":{"total":767960285184,"available":767731142656,"internally_reserved":0,"allocated":7790592,"data_stored":4373331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14248,"internal_metadata":221235288},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004741"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.478251+0000","last_change":"2024-03-07T03:11:17.762433+0000","last_active":"2024-03-07T03:11:49.478251+0000","last_peered":"2024-03-07T03:11:49.478251+0000","last_clean":"2024-03-07T03:11:49.478251+0000","last_became_active":"2024-03-07T03:11:17.761957+0000","last_became_peered":"2024-03-07T03:11:17.761957+0000","last_unstale":"2024-03-07T03:11:49.478251+0000","last_undegraded":"2024-03-07T03:11:49.478251+0000","last_fullsized":"2024-03-07T03:11:49.478251+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:22:22.662579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868315+0000","last_change":"2024-03-07T03:11:17.752329+0000","last_active":"2024-03-07T03:11:49.868315+0000","last_peered":"2024-03-07T03:11:49.868315+0000","last_clean":"2024-03-07T03:11:49.868315+0000","last_became_active":"2024-03-07T03:11:17.751695+0000","last_became_peered":"2024-03-07T03:11:17.751695+0000","last_unstale":"2024-03-07T03:11:49.868315+0000","last_undegraded":"2024-03-07T03:11:49.868315+0000","last_fullsized":"2024-03-07T03:11:49.868315+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:31:05.696543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479282+0000","last_change":"2024-03-07T03:11:17.844094+0000","last_active":"2024-03-07T03:11:49.479282+0000","last_peered":"2024-03-07T03:11:49.479282+0000","last_clean":"2024-03-07T03:11:49.479282+0000","last_became_active":"2024-03-07T03:11:17.843769+0000","last_became_peered":"2024-03-07T03:11:17.843769+0000","last_unstale":"2024-03-07T03:11:49.479282+0000","last_undegraded":"2024-03-07T03:11:49.479282+0000","last_fullsized":"2024-03-07T03:11:49.479282+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:20:13.146211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477682+0000","last_change":"2024-03-07T03:11:17.733223+0000","last_active":"2024-03-07T03:11:49.477682+0000","last_peered":"2024-03-07T03:11:49.477682+0000","last_clean":"2024-03-07T03:11:49.477682+0000","last_became_active":"2024-03-07T03:11:17.731893+0000","last_became_peered":"2024-03-07T03:11:17.731893+0000","last_unstale":"2024-03-07T03:11:49.477682+0000","last_undegraded":"2024-03-07T03:11:49.477682+0000","last_fullsized":"2024-03-07T03:11:49.477682+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:21:12.888076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480674+0000","last_change":"2024-03-07T03:11:17.763532+0000","last_active":"2024-03-07T03:11:49.480674+0000","last_peered":"2024-03-07T03:11:49.480674+0000","last_clean":"2024-03-07T03:11:49.480674+0000","last_became_active":"2024-03-07T03:11:17.763206+0000","last_became_peered":"2024-03-07T03:11:17.763206+0000","last_unstale":"2024-03-07T03:11:49.480674+0000","last_undegraded":"2024-03-07T03:11:49.480674+0000","last_fullsized":"2024-03-07T03:11:49.480674+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:04:50.514780+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480802+0000","last_change":"2024-03-07T03:11:17.728694+0000","last_active":"2024-03-07T03:11:49.480802+0000","last_peered":"2024-03-07T03:11:49.480802+0000","last_clean":"2024-03-07T03:11:49.480802+0000","last_became_active":"2024-03-07T03:11:17.727704+0000","last_became_peered":"2024-03-07T03:11:17.727704+0000","last_unstale":"2024-03-07T03:11:49.480802+0000","last_undegraded":"2024-03-07T03:11:49.480802+0000","last_fullsized":"2024-03-07T03:11:49.480802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:18:04.455845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.481532+0000","last_change":"2024-03-07T03:11:17.763765+0000","last_active":"2024-03-07T03:11:49.481532+0000","last_peered":"2024-03-07T03:11:49.481532+0000","last_clean":"2024-03-07T03:11:49.481532+0000","last_became_active":"2024-03-07T03:11:17.763361+0000","last_became_peered":"2024-03-07T03:11:17.763361+0000","last_unstale":"2024-03-07T03:11:49.481532+0000","last_undegraded":"2024-03-07T03:11:49.481532+0000","last_fullsized":"2024-03-07T03:11:49.481532+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:42:48.387310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480807+0000","last_change":"2024-03-07T03:11:17.726915+0000","last_active":"2024-03-07T03:11:49.480807+0000","last_peered":"2024-03-07T03:11:49.480807+0000","last_clean":"2024-03-07T03:11:49.480807+0000","last_became_active":"2024-03-07T03:11:17.726078+0000","last_became_peered":"2024-03-07T03:11:17.726078+0000","last_unstale":"2024-03-07T03:11:49.480807+0000","last_undegraded":"2024-03-07T03:11:49.480807+0000","last_fullsized":"2024-03-07T03:11:49.480807+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:00:52.554381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477939+0000","last_change":"2024-03-07T03:11:19.028055+0000","last_active":"2024-03-07T03:11:49.477939+0000","last_peered":"2024-03-07T03:11:49.477939+0000","last_clean":"2024-03-07T03:11:49.477939+0000","last_became_active":"2024-03-07T03:11:19.026706+0000","last_became_peered":"2024-03-07T03:11:19.026706+0000","last_unstale":"2024-03-07T03:11:49.477939+0000","last_undegraded":"2024-03-07T03:11:49.477939+0000","last_fullsized":"2024-03-07T03:11:49.477939+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:27:09.803901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868264+0000","last_change":"2024-03-07T03:11:17.730681+0000","last_active":"2024-03-07T03:11:49.868264+0000","last_peered":"2024-03-07T03:11:49.868264+0000","last_clean":"2024-03-07T03:11:49.868264+0000","last_became_active":"2024-03-07T03:11:17.729900+0000","last_became_peered":"2024-03-07T03:11:17.729900+0000","last_unstale":"2024-03-07T03:11:49.868264+0000","last_undegraded":"2024-03-07T03:11:49.868264+0000","last_fullsized":"2024-03-07T03:11:49.868264+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:43:00.084654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482328+0000","last_change":"2024-03-07T03:11:17.753267+0000","last_active":"2024-03-07T03:11:49.482328+0000","last_peered":"2024-03-07T03:11:49.482328+0000","last_clean":"2024-03-07T03:11:49.482328+0000","last_became_active":"2024-03-07T03:11:17.752977+0000","last_became_peered":"2024-03-07T03:11:17.752977+0000","last_unstale":"2024-03-07T03:11:49.482328+0000","last_undegraded":"2024-03-07T03:11:49.482328+0000","last_fullsized":"2024-03-07T03:11:49.482328+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:00:24.404134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868429+0000","last_change":"2024-03-07T03:11:17.735077+0000","last_active":"2024-03-07T03:11:49.868429+0000","last_peered":"2024-03-07T03:11:49.868429+0000","last_clean":"2024-03-07T03:11:49.868429+0000","last_became_active":"2024-03-07T03:11:17.731073+0000","last_became_peered":"2024-03-07T03:11:17.731073+0000","last_unstale":"2024-03-07T03:11:49.868429+0000","last_undegraded":"2024-03-07T03:11:49.868429+0000","last_fullsized":"2024-03-07T03:11:49.868429+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:08:07.371789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480953+0000","last_change":"2024-03-07T03:09:31.339012+0000","last_active":"2024-03-07T03:11:49.480953+0000","last_peered":"2024-03-07T03:11:49.480953+0000","last_clean":"2024-03-07T03:11:49.480953+0000","last_became_active":"2024-03-07T03:09:31.338487+0000","last_became_peered":"2024-03-07T03:09:31.338487+0000","last_unstale":"2024-03-07T03:11:49.480953+0000","last_undegraded":"2024-03-07T03:11:49.480953+0000","last_fullsized":"2024-03-07T03:11:49.480953+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:48:57.153468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.474725+0000","last_change":"2024-03-07T03:11:17.850632+0000","last_active":"2024-03-07T03:11:49.474725+0000","last_peered":"2024-03-07T03:11:49.474725+0000","last_clean":"2024-03-07T03:11:49.474725+0000","last_became_active":"2024-03-07T03:11:17.850392+0000","last_became_peered":"2024-03-07T03:11:17.850392+0000","last_unstale":"2024-03-07T03:11:49.474725+0000","last_undegraded":"2024-03-07T03:11:49.474725+0000","last_fullsized":"2024-03-07T03:11:49.474725+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:41:12.881887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487226+0000","last_change":"2024-03-07T03:09:31.492237+0000","last_active":"2024-03-07T03:11:49.487226+0000","last_peered":"2024-03-07T03:11:49.487226+0000","last_clean":"2024-03-07T03:11:49.487226+0000","last_became_active":"2024-03-07T03:09:31.490659+0000","last_became_peered":"2024-03-07T03:09:31.490659+0000","last_unstale":"2024-03-07T03:11:49.487226+0000","last_undegraded":"2024-03-07T03:11:49.487226+0000","last_fullsized":"2024-03-07T03:11:49.487226+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:58:14.548445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480366+0000","last_change":"2024-03-07T03:11:17.744707+0000","last_active":"2024-03-07T03:11:49.480366+0000","last_peered":"2024-03-07T03:11:49.480366+0000","last_clean":"2024-03-07T03:11:49.480366+0000","last_became_active":"2024-03-07T03:11:17.744274+0000","last_became_peered":"2024-03-07T03:11:17.744274+0000","last_unstale":"2024-03-07T03:11:49.480366+0000","last_undegraded":"2024-03-07T03:11:49.480366+0000","last_fullsized":"2024-03-07T03:11:49.480366+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:40:12.704059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869230+0000","last_change":"2024-03-07T03:11:19.034716+0000","last_active":"2024-03-07T03:11:49.869230+0000","last_peered":"2024-03-07T03:11:49.869230+0000","last_clean":"2024-03-07T03:11:49.869230+0000","last_became_active":"2024-03-07T03:11:19.032902+0000","last_became_peered":"2024-03-07T03:11:19.032902+0000","last_unstale":"2024-03-07T03:11:49.869230+0000","last_undegraded":"2024-03-07T03:11:49.869230+0000","last_fullsized":"2024-03-07T03:11:49.869230+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:38:08.075495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487222+0000","last_change":"2024-03-07T03:11:17.737252+0000","last_active":"2024-03-07T03:11:49.487222+0000","last_peered":"2024-03-07T03:11:49.487222+0000","last_clean":"2024-03-07T03:11:49.487222+0000","last_became_active":"2024-03-07T03:11:17.736246+0000","last_became_peered":"2024-03-07T03:11:17.736246+0000","last_unstale":"2024-03-07T03:11:49.487222+0000","last_undegraded":"2024-03-07T03:11:49.487222+0000","last_fullsized":"2024-03-07T03:11:49.487222+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:49:42.107373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473523+0000","last_change":"2024-03-07T03:09:31.490148+0000","last_active":"2024-03-07T03:11:49.473523+0000","last_peered":"2024-03-07T03:11:49.473523+0000","last_clean":"2024-03-07T03:11:49.473523+0000","last_became_active":"2024-03-07T03:09:31.489708+0000","last_became_peered":"2024-03-07T03:09:31.489708+0000","last_unstale":"2024-03-07T03:11:49.473523+0000","last_undegraded":"2024-03-07T03:11:49.473523+0000","last_fullsized":"2024-03-07T03:11:49.473523+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:21:06.820876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869757+0000","last_change":"2024-03-07T03:11:19.029566+0000","last_active":"2024-03-07T03:11:49.869757+0000","last_peered":"2024-03-07T03:11:49.869757+0000","last_clean":"2024-03-07T03:11:49.869757+0000","last_became_active":"2024-03-07T03:11:19.028550+0000","last_became_peered":"2024-03-07T03:11:19.028550+0000","last_unstale":"2024-03-07T03:11:49.869757+0000","last_undegraded":"2024-03-07T03:11:49.869757+0000","last_fullsized":"2024-03-07T03:11:49.869757+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:35:52.637946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.867810+0000","last_change":"2024-03-07T03:11:17.741573+0000","last_active":"2024-03-07T03:11:49.867810+0000","last_peered":"2024-03-07T03:11:49.867810+0000","last_clean":"2024-03-07T03:11:49.867810+0000","last_became_active":"2024-03-07T03:11:17.736518+0000","last_became_peered":"2024-03-07T03:11:17.736518+0000","last_unstale":"2024-03-07T03:11:49.867810+0000","last_undegraded":"2024-03-07T03:11:49.867810+0000","last_fullsized":"2024-03-07T03:11:49.867810+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:47:03.005123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.472243+0000","last_change":"2024-03-07T03:09:31.490070+0000","last_active":"2024-03-07T03:11:49.472243+0000","last_peered":"2024-03-07T03:11:49.472243+0000","last_clean":"2024-03-07T03:11:49.472243+0000","last_became_active":"2024-03-07T03:09:31.489632+0000","last_became_peered":"2024-03-07T03:09:31.489632+0000","last_unstale":"2024-03-07T03:11:49.472243+0000","last_undegraded":"2024-03-07T03:11:49.472243+0000","last_fullsized":"2024-03-07T03:11:49.472243+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:08:00.120410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482066+0000","last_change":"2024-03-07T03:11:17.849560+0000","last_active":"2024-03-07T03:11:49.482066+0000","last_peered":"2024-03-07T03:11:49.482066+0000","last_clean":"2024-03-07T03:11:49.482066+0000","last_became_active":"2024-03-07T03:11:17.848681+0000","last_became_peered":"2024-03-07T03:11:17.848681+0000","last_unstale":"2024-03-07T03:11:49.482066+0000","last_undegraded":"2024-03-07T03:11:49.482066+0000","last_fullsized":"2024-03-07T03:11:49.482066+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:29:42.167986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475798+0000","last_change":"2024-03-07T03:11:17.727584+0000","last_active":"2024-03-07T03:11:49.475798+0000","last_peered":"2024-03-07T03:11:49.475798+0000","last_clean":"2024-03-07T03:11:49.475798+0000","last_became_active":"2024-03-07T03:11:17.726901+0000","last_became_peered":"2024-03-07T03:11:17.726901+0000","last_unstale":"2024-03-07T03:11:49.475798+0000","last_undegraded":"2024-03-07T03:11:49.475798+0000","last_fullsized":"2024-03-07T03:11:49.475798+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:12:30.904702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":269,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:50.656673+0000","last_change":"2024-03-07T03:07:57.209434+0000","last_active":"2024-03-07T03:11:50.656673+0000","last_peered":"2024-03-07T03:11:50.656673+0000","last_clean":"2024-03-07T03:11:50.656673+0000","last_became_active":"2024-03-07T03:07:57.208461+0000","last_became_peered":"2024-03-07T03:07:57.208461+0000","last_unstale":"2024-03-07T03:11:50.656673+0000","last_undegraded":"2024-03-07T03:11:50.656673+0000","last_fullsized":"2024-03-07T03:11:50.656673+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_clean_scrub_stamp":"2024-03-07T03:03:44.169894+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:55:26.915133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479911+0000","last_change":"2024-03-07T03:09:31.338124+0000","last_active":"2024-03-07T03:11:49.479911+0000","last_peered":"2024-03-07T03:11:49.479911+0000","last_clean":"2024-03-07T03:11:49.479911+0000","last_became_active":"2024-03-07T03:09:31.337531+0000","last_became_peered":"2024-03-07T03:09:31.337531+0000","last_unstale":"2024-03-07T03:11:49.479911+0000","last_undegraded":"2024-03-07T03:11:49.479911+0000","last_fullsized":"2024-03-07T03:11:49.479911+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:39.200015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487610+0000","last_change":"2024-03-07T03:11:17.768359+0000","last_active":"2024-03-07T03:11:49.487610+0000","last_peered":"2024-03-07T03:11:49.487610+0000","last_clean":"2024-03-07T03:11:49.487610+0000","last_became_active":"2024-03-07T03:11:17.767774+0000","last_became_peered":"2024-03-07T03:11:17.767774+0000","last_unstale":"2024-03-07T03:11:49.487610+0000","last_undegraded":"2024-03-07T03:11:49.487610+0000","last_fullsized":"2024-03-07T03:11:49.487610+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:20:11.902070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485106+0000","last_change":"2024-03-07T03:11:17.746565+0000","last_active":"2024-03-07T03:11:49.485106+0000","last_peered":"2024-03-07T03:11:49.485106+0000","last_clean":"2024-03-07T03:11:49.485106+0000","last_became_active":"2024-03-07T03:11:17.744126+0000","last_became_peered":"2024-03-07T03:11:17.744126+0000","last_unstale":"2024-03-07T03:11:49.485106+0000","last_undegraded":"2024-03-07T03:11:49.485106+0000","last_fullsized":"2024-03-07T03:11:49.485106+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:14:03.441173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477100+0000","last_change":"2024-03-07T03:09:31.492946+0000","last_active":"2024-03-07T03:11:49.477100+0000","last_peered":"2024-03-07T03:11:49.477100+0000","last_clean":"2024-03-07T03:11:49.477100+0000","last_became_active":"2024-03-07T03:09:31.491949+0000","last_became_peered":"2024-03-07T03:09:31.491949+0000","last_unstale":"2024-03-07T03:11:49.477100+0000","last_undegraded":"2024-03-07T03:11:49.477100+0000","last_fullsized":"2024-03-07T03:11:49.477100+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:42:06.966194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":11,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:51.152125+0000","last_change":"2024-03-07T03:11:51.152125+0000","last_active":"2024-03-07T03:11:51.152125+0000","last_peered":"2024-03-07T03:11:51.152125+0000","last_clean":"2024-03-07T03:11:51.152125+0000","last_became_active":"2024-03-07T03:11:51.150853+0000","last_became_peered":"2024-03-07T03:11:51.150853+0000","last_unstale":"2024-03-07T03:11:51.152125+0000","last_undegraded":"2024-03-07T03:11:51.152125+0000","last_fullsized":"2024-03-07T03:11:51.152125+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:20:44.149315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.481289+0000","last_change":"2024-03-07T03:11:17.734091+0000","last_active":"2024-03-07T03:11:49.481289+0000","last_peered":"2024-03-07T03:11:49.481289+0000","last_clean":"2024-03-07T03:11:49.481289+0000","last_became_active":"2024-03-07T03:11:17.733811+0000","last_became_peered":"2024-03-07T03:11:17.733811+0000","last_unstale":"2024-03-07T03:11:49.481289+0000","last_undegraded":"2024-03-07T03:11:49.481289+0000","last_fullsized":"2024-03-07T03:11:49.481289+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:35:23.691468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479579+0000","last_change":"2024-03-07T03:09:31.340350+0000","last_active":"2024-03-07T03:11:49.479579+0000","last_peered":"2024-03-07T03:11:49.479579+0000","last_clean":"2024-03-07T03:11:49.479579+0000","last_became_active":"2024-03-07T03:09:31.339524+0000","last_became_peered":"2024-03-07T03:09:31.339524+0000","last_unstale":"2024-03-07T03:11:49.479579+0000","last_undegraded":"2024-03-07T03:11:49.479579+0000","last_fullsized":"2024-03-07T03:11:49.479579+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:14:32.989838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485520+0000","last_change":"2024-03-07T03:11:17.763549+0000","last_active":"2024-03-07T03:11:49.485520+0000","last_peered":"2024-03-07T03:11:49.485520+0000","last_clean":"2024-03-07T03:11:49.485520+0000","last_became_active":"2024-03-07T03:11:17.762902+0000","last_became_peered":"2024-03-07T03:11:17.762902+0000","last_unstale":"2024-03-07T03:11:49.485520+0000","last_undegraded":"2024-03-07T03:11:49.485520+0000","last_fullsized":"2024-03-07T03:11:49.485520+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:24:59.291166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480521+0000","last_change":"2024-03-07T03:11:17.736543+0000","last_active":"2024-03-07T03:11:49.480521+0000","last_peered":"2024-03-07T03:11:49.480521+0000","last_clean":"2024-03-07T03:11:49.480521+0000","last_became_active":"2024-03-07T03:11:17.735167+0000","last_became_peered":"2024-03-07T03:11:17.735167+0000","last_unstale":"2024-03-07T03:11:49.480521+0000","last_undegraded":"2024-03-07T03:11:49.480521+0000","last_fullsized":"2024-03-07T03:11:49.480521+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:06:39.189368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.486988+0000","last_change":"2024-03-07T03:09:31.493118+0000","last_active":"2024-03-07T03:11:49.486988+0000","last_peered":"2024-03-07T03:11:49.486988+0000","last_clean":"2024-03-07T03:11:49.486988+0000","last_became_active":"2024-03-07T03:09:31.491952+0000","last_became_peered":"2024-03-07T03:09:31.491952+0000","last_unstale":"2024-03-07T03:11:49.486988+0000","last_undegraded":"2024-03-07T03:11:49.486988+0000","last_fullsized":"2024-03-07T03:11:49.486988+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:52:26.178237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.487689+0000","last_change":"2024-03-07T03:11:17.762401+0000","last_active":"2024-03-07T03:11:49.487689+0000","last_peered":"2024-03-07T03:11:49.487689+0000","last_clean":"2024-03-07T03:11:49.487689+0000","last_became_active":"2024-03-07T03:11:17.761846+0000","last_became_peered":"2024-03-07T03:11:17.761846+0000","last_unstale":"2024-03-07T03:11:49.487689+0000","last_undegraded":"2024-03-07T03:11:49.487689+0000","last_fullsized":"2024-03-07T03:11:49.487689+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:54:19.206136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.474837+0000","last_change":"2024-03-07T03:11:17.722629+0000","last_active":"2024-03-07T03:11:49.474837+0000","last_peered":"2024-03-07T03:11:49.474837+0000","last_clean":"2024-03-07T03:11:49.474837+0000","last_became_active":"2024-03-07T03:11:17.721320+0000","last_became_peered":"2024-03-07T03:11:17.721320+0000","last_unstale":"2024-03-07T03:11:49.474837+0000","last_undegraded":"2024-03-07T03:11:49.474837+0000","last_fullsized":"2024-03-07T03:11:49.474837+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:29:11.930358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.483483+0000","last_change":"2024-03-07T03:11:17.763117+0000","last_active":"2024-03-07T03:11:49.483483+0000","last_peered":"2024-03-07T03:11:49.483483+0000","last_clean":"2024-03-07T03:11:49.483483+0000","last_became_active":"2024-03-07T03:11:17.761698+0000","last_became_peered":"2024-03-07T03:11:17.761698+0000","last_unstale":"2024-03-07T03:11:49.483483+0000","last_undegraded":"2024-03-07T03:11:49.483483+0000","last_fullsized":"2024-03-07T03:11:49.483483+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:34:22.206821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482449+0000","last_change":"2024-03-07T03:11:17.736365+0000","last_active":"2024-03-07T03:11:49.482449+0000","last_peered":"2024-03-07T03:11:49.482449+0000","last_clean":"2024-03-07T03:11:49.482449+0000","last_became_active":"2024-03-07T03:11:17.728478+0000","last_became_peered":"2024-03-07T03:11:17.728478+0000","last_unstale":"2024-03-07T03:11:49.482449+0000","last_undegraded":"2024-03-07T03:11:49.482449+0000","last_fullsized":"2024-03-07T03:11:49.482449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:24:39.629084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:51.152721+0000","last_change":"2024-03-07T03:11:51.152721+0000","last_active":"2024-03-07T03:11:51.152721+0000","last_peered":"2024-03-07T03:11:51.152721+0000","last_clean":"2024-03-07T03:11:51.152721+0000","last_became_active":"2024-03-07T03:11:51.151718+0000","last_became_peered":"2024-03-07T03:11:51.151718+0000","last_unstale":"2024-03-07T03:11:51.152721+0000","last_undegraded":"2024-03-07T03:11:51.152721+0000","last_fullsized":"2024-03-07T03:11:51.152721+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:16:19.214983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480726+0000","last_change":"2024-03-07T03:11:17.727286+0000","last_active":"2024-03-07T03:11:49.480726+0000","last_peered":"2024-03-07T03:11:49.480726+0000","last_clean":"2024-03-07T03:11:49.480726+0000","last_became_active":"2024-03-07T03:11:17.726975+0000","last_became_peered":"2024-03-07T03:11:17.726975+0000","last_unstale":"2024-03-07T03:11:49.480726+0000","last_undegraded":"2024-03-07T03:11:49.480726+0000","last_fullsized":"2024-03-07T03:11:49.480726+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:46:19.775858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480492+0000","last_change":"2024-03-07T03:11:17.762607+0000","last_active":"2024-03-07T03:11:49.480492+0000","last_peered":"2024-03-07T03:11:49.480492+0000","last_clean":"2024-03-07T03:11:49.480492+0000","last_became_active":"2024-03-07T03:11:17.761393+0000","last_became_peered":"2024-03-07T03:11:17.761393+0000","last_unstale":"2024-03-07T03:11:49.480492+0000","last_undegraded":"2024-03-07T03:11:49.480492+0000","last_fullsized":"2024-03-07T03:11:49.480492+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:38:34.500610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475557+0000","last_change":"2024-03-07T03:11:17.745662+0000","last_active":"2024-03-07T03:11:49.475557+0000","last_peered":"2024-03-07T03:11:49.475557+0000","last_clean":"2024-03-07T03:11:49.475557+0000","last_became_active":"2024-03-07T03:11:17.745370+0000","last_became_peered":"2024-03-07T03:11:17.745370+0000","last_unstale":"2024-03-07T03:11:49.475557+0000","last_undegraded":"2024-03-07T03:11:49.475557+0000","last_fullsized":"2024-03-07T03:11:49.475557+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:58:02.187340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869070+0000","last_change":"2024-03-07T03:11:19.035152+0000","last_active":"2024-03-07T03:11:49.869070+0000","last_peered":"2024-03-07T03:11:49.869070+0000","last_clean":"2024-03-07T03:11:49.869070+0000","last_became_active":"2024-03-07T03:11:19.033898+0000","last_became_peered":"2024-03-07T03:11:19.033898+0000","last_unstale":"2024-03-07T03:11:49.869070+0000","last_undegraded":"2024-03-07T03:11:49.869070+0000","last_fullsized":"2024-03-07T03:11:49.869070+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:39:57.234259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480635+0000","last_change":"2024-03-07T03:11:17.722831+0000","last_active":"2024-03-07T03:11:49.480635+0000","last_peered":"2024-03-07T03:11:49.480635+0000","last_clean":"2024-03-07T03:11:49.480635+0000","last_became_active":"2024-03-07T03:11:17.720768+0000","last_became_peered":"2024-03-07T03:11:17.720768+0000","last_unstale":"2024-03-07T03:11:49.480635+0000","last_undegraded":"2024-03-07T03:11:49.480635+0000","last_fullsized":"2024-03-07T03:11:49.480635+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:11:28.934497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-03-07T03:11:51.153475+0000","last_change":"2024-03-07T03:11:51.153475+0000","last_active":"2024-03-07T03:11:51.153475+0000","last_peered":"2024-03-07T03:11:51.153475+0000","last_clean":"2024-03-07T03:11:51.153475+0000","last_became_active":"2024-03-07T03:11:51.152067+0000","last_became_peered":"2024-03-07T03:11:51.152067+0000","last_unstale":"2024-03-07T03:11:51.153475+0000","last_undegraded":"2024-03-07T03:11:51.153475+0000","last_fullsized":"2024-03-07T03:11:51.153475+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:28:48.025654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473642+0000","last_change":"2024-03-07T03:11:17.738764+0000","last_active":"2024-03-07T03:11:49.473642+0000","last_peered":"2024-03-07T03:11:49.473642+0000","last_clean":"2024-03-07T03:11:49.473642+0000","last_became_active":"2024-03-07T03:11:17.738151+0000","last_became_peered":"2024-03-07T03:11:17.738151+0000","last_unstale":"2024-03-07T03:11:49.473642+0000","last_undegraded":"2024-03-07T03:11:49.473642+0000","last_fullsized":"2024-03-07T03:11:49.473642+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:37:22.289110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482158+0000","last_change":"2024-03-07T03:11:17.762914+0000","last_active":"2024-03-07T03:11:49.482158+0000","last_peered":"2024-03-07T03:11:49.482158+0000","last_clean":"2024-03-07T03:11:49.482158+0000","last_became_active":"2024-03-07T03:11:17.762339+0000","last_became_peered":"2024-03-07T03:11:17.762339+0000","last_unstale":"2024-03-07T03:11:49.482158+0000","last_undegraded":"2024-03-07T03:11:49.482158+0000","last_fullsized":"2024-03-07T03:11:49.482158+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:01:57.515149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473161+0000","last_change":"2024-03-07T03:11:17.737593+0000","last_active":"2024-03-07T03:11:49.473161+0000","last_peered":"2024-03-07T03:11:49.473161+0000","last_clean":"2024-03-07T03:11:49.473161+0000","last_became_active":"2024-03-07T03:11:17.732115+0000","last_became_peered":"2024-03-07T03:11:17.732115+0000","last_unstale":"2024-03-07T03:11:49.473161+0000","last_undegraded":"2024-03-07T03:11:49.473161+0000","last_fullsized":"2024-03-07T03:11:49.473161+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:31:45.710698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479259+0000","last_change":"2024-03-07T03:11:17.846593+0000","last_active":"2024-03-07T03:11:49.479259+0000","last_peered":"2024-03-07T03:11:49.479259+0000","last_clean":"2024-03-07T03:11:49.479259+0000","last_became_active":"2024-03-07T03:11:17.845734+0000","last_became_peered":"2024-03-07T03:11:17.845734+0000","last_unstale":"2024-03-07T03:11:49.479259+0000","last_undegraded":"2024-03-07T03:11:49.479259+0000","last_fullsized":"2024-03-07T03:11:49.479259+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:18:40.161672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473071+0000","last_change":"2024-03-07T03:11:17.735133+0000","last_active":"2024-03-07T03:11:49.473071+0000","last_peered":"2024-03-07T03:11:49.473071+0000","last_clean":"2024-03-07T03:11:49.473071+0000","last_became_active":"2024-03-07T03:11:17.734029+0000","last_became_peered":"2024-03-07T03:11:17.734029+0000","last_unstale":"2024-03-07T03:11:49.473071+0000","last_undegraded":"2024-03-07T03:11:49.473071+0000","last_fullsized":"2024-03-07T03:11:49.473071+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:51:27.606846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475621+0000","last_change":"2024-03-07T03:11:17.847015+0000","last_active":"2024-03-07T03:11:49.475621+0000","last_peered":"2024-03-07T03:11:49.475621+0000","last_clean":"2024-03-07T03:11:49.475621+0000","last_became_active":"2024-03-07T03:11:17.846740+0000","last_became_peered":"2024-03-07T03:11:17.846740+0000","last_unstale":"2024-03-07T03:11:49.475621+0000","last_undegraded":"2024-03-07T03:11:49.475621+0000","last_fullsized":"2024-03-07T03:11:49.475621+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:51:17.128266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":32,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477033+0000","last_change":"2024-03-07T03:11:17.750812+0000","last_active":"2024-03-07T03:11:49.477033+0000","last_peered":"2024-03-07T03:11:49.477033+0000","last_clean":"2024-03-07T03:11:49.477033+0000","last_became_active":"2024-03-07T03:11:17.750471+0000","last_became_peered":"2024-03-07T03:11:17.750471+0000","last_unstale":"2024-03-07T03:11:49.477033+0000","last_undegraded":"2024-03-07T03:11:49.477033+0000","last_fullsized":"2024-03-07T03:11:49.477033+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:58:02.992427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869340+0000","last_change":"2024-03-07T03:11:19.031805+0000","last_active":"2024-03-07T03:11:49.869340+0000","last_peered":"2024-03-07T03:11:49.869340+0000","last_clean":"2024-03-07T03:11:49.869340+0000","last_became_active":"2024-03-07T03:11:19.031219+0000","last_became_peered":"2024-03-07T03:11:19.031219+0000","last_unstale":"2024-03-07T03:11:49.869340+0000","last_undegraded":"2024-03-07T03:11:49.869340+0000","last_fullsized":"2024-03-07T03:11:49.869340+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:25:58.130337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.472079+0000","last_change":"2024-03-07T03:11:17.733218+0000","last_active":"2024-03-07T03:11:49.472079+0000","last_peered":"2024-03-07T03:11:49.472079+0000","last_clean":"2024-03-07T03:11:49.472079+0000","last_became_active":"2024-03-07T03:11:17.732035+0000","last_became_peered":"2024-03-07T03:11:17.732035+0000","last_unstale":"2024-03-07T03:11:49.472079+0000","last_undegraded":"2024-03-07T03:11:49.472079+0000","last_fullsized":"2024-03-07T03:11:49.472079+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:29:32.508319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480279+0000","last_change":"2024-03-07T03:11:17.849679+0000","last_active":"2024-03-07T03:11:49.480279+0000","last_peered":"2024-03-07T03:11:49.480279+0000","last_clean":"2024-03-07T03:11:49.480279+0000","last_became_active":"2024-03-07T03:11:17.848732+0000","last_became_peered":"2024-03-07T03:11:17.848732+0000","last_unstale":"2024-03-07T03:11:49.480279+0000","last_undegraded":"2024-03-07T03:11:49.480279+0000","last_fullsized":"2024-03-07T03:11:49.480279+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:32:32.048244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868028+0000","last_change":"2024-03-07T03:11:17.729899+0000","last_active":"2024-03-07T03:11:49.868028+0000","last_peered":"2024-03-07T03:11:49.868028+0000","last_clean":"2024-03-07T03:11:49.868028+0000","last_became_active":"2024-03-07T03:11:17.729108+0000","last_became_peered":"2024-03-07T03:11:17.729108+0000","last_unstale":"2024-03-07T03:11:49.868028+0000","last_undegraded":"2024-03-07T03:11:49.868028+0000","last_fullsized":"2024-03-07T03:11:49.868028+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:31:13.744056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.477933+0000","last_change":"2024-03-07T03:11:17.846954+0000","last_active":"2024-03-07T03:11:49.477933+0000","last_peered":"2024-03-07T03:11:49.477933+0000","last_clean":"2024-03-07T03:11:49.477933+0000","last_became_active":"2024-03-07T03:11:17.846652+0000","last_became_peered":"2024-03-07T03:11:17.846652+0000","last_unstale":"2024-03-07T03:11:49.477933+0000","last_undegraded":"2024-03-07T03:11:49.477933+0000","last_fullsized":"2024-03-07T03:11:49.477933+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:18:30.656475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.472893+0000","last_change":"2024-03-07T03:11:17.730834+0000","last_active":"2024-03-07T03:11:49.472893+0000","last_peered":"2024-03-07T03:11:49.472893+0000","last_clean":"2024-03-07T03:11:49.472893+0000","last_became_active":"2024-03-07T03:11:17.730541+0000","last_became_peered":"2024-03-07T03:11:17.730541+0000","last_unstale":"2024-03-07T03:11:49.472893+0000","last_undegraded":"2024-03-07T03:11:49.472893+0000","last_fullsized":"2024-03-07T03:11:49.472893+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:41:51.335682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480845+0000","last_change":"2024-03-07T03:11:17.762938+0000","last_active":"2024-03-07T03:11:49.480845+0000","last_peered":"2024-03-07T03:11:49.480845+0000","last_clean":"2024-03-07T03:11:49.480845+0000","last_became_active":"2024-03-07T03:11:17.762293+0000","last_became_peered":"2024-03-07T03:11:17.762293+0000","last_unstale":"2024-03-07T03:11:49.480845+0000","last_undegraded":"2024-03-07T03:11:49.480845+0000","last_fullsized":"2024-03-07T03:11:49.480845+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:47:14.683735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":29,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.482881+0000","last_change":"2024-03-07T03:11:17.721344+0000","last_active":"2024-03-07T03:11:49.482881+0000","last_peered":"2024-03-07T03:11:49.482881+0000","last_clean":"2024-03-07T03:11:49.482881+0000","last_became_active":"2024-03-07T03:11:17.720857+0000","last_became_peered":"2024-03-07T03:11:17.720857+0000","last_unstale":"2024-03-07T03:11:49.482881+0000","last_undegraded":"2024-03-07T03:11:49.482881+0000","last_fullsized":"2024-03-07T03:11:49.482881+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:49:14.558664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485350+0000","last_change":"2024-03-07T03:11:17.844288+0000","last_active":"2024-03-07T03:11:49.485350+0000","last_peered":"2024-03-07T03:11:49.485350+0000","last_clean":"2024-03-07T03:11:49.485350+0000","last_became_active":"2024-03-07T03:11:17.843897+0000","last_became_peered":"2024-03-07T03:11:17.843897+0000","last_unstale":"2024-03-07T03:11:49.485350+0000","last_undegraded":"2024-03-07T03:11:49.485350+0000","last_fullsized":"2024-03-07T03:11:49.485350+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:48:48.959645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.473409+0000","last_change":"2024-03-07T03:11:17.754277+0000","last_active":"2024-03-07T03:11:49.473409+0000","last_peered":"2024-03-07T03:11:49.473409+0000","last_clean":"2024-03-07T03:11:49.473409+0000","last_became_active":"2024-03-07T03:11:17.753979+0000","last_became_peered":"2024-03-07T03:11:17.753979+0000","last_unstale":"2024-03-07T03:11:49.473409+0000","last_undegraded":"2024-03-07T03:11:49.473409+0000","last_fullsized":"2024-03-07T03:11:49.473409+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:00:02.786312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.479516+0000","last_change":"2024-03-07T03:11:17.850841+0000","last_active":"2024-03-07T03:11:49.479516+0000","last_peered":"2024-03-07T03:11:49.479516+0000","last_clean":"2024-03-07T03:11:49.479516+0000","last_became_active":"2024-03-07T03:11:17.850452+0000","last_became_peered":"2024-03-07T03:11:17.850452+0000","last_unstale":"2024-03-07T03:11:49.479516+0000","last_undegraded":"2024-03-07T03:11:49.479516+0000","last_fullsized":"2024-03-07T03:11:49.479516+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:11:24.249898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.481058+0000","last_change":"2024-03-07T03:11:17.731035+0000","last_active":"2024-03-07T03:11:49.481058+0000","last_peered":"2024-03-07T03:11:49.481058+0000","last_clean":"2024-03-07T03:11:49.481058+0000","last_became_active":"2024-03-07T03:11:17.730092+0000","last_became_peered":"2024-03-07T03:11:17.730092+0000","last_unstale":"2024-03-07T03:11:49.481058+0000","last_undegraded":"2024-03-07T03:11:49.481058+0000","last_fullsized":"2024-03-07T03:11:49.481058+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:20:50.529165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475051+0000","last_change":"2024-03-07T03:11:17.761532+0000","last_active":"2024-03-07T03:11:49.475051+0000","last_peered":"2024-03-07T03:11:49.475051+0000","last_clean":"2024-03-07T03:11:49.475051+0000","last_became_active":"2024-03-07T03:11:17.761092+0000","last_became_peered":"2024-03-07T03:11:17.761092+0000","last_unstale":"2024-03-07T03:11:49.475051+0000","last_undegraded":"2024-03-07T03:11:49.475051+0000","last_fullsized":"2024-03-07T03:11:49.475051+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:51:19.912304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.868469+0000","last_change":"2024-03-07T03:11:17.750987+0000","last_active":"2024-03-07T03:11:49.868469+0000","last_peered":"2024-03-07T03:11:49.868469+0000","last_clean":"2024-03-07T03:11:49.868469+0000","last_became_active":"2024-03-07T03:11:17.750258+0000","last_became_peered":"2024-03-07T03:11:17.750258+0000","last_unstale":"2024-03-07T03:11:49.868469+0000","last_undegraded":"2024-03-07T03:11:49.868469+0000","last_fullsized":"2024-03-07T03:11:49.868469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:40:37.611600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.478345+0000","last_change":"2024-03-07T03:11:17.758817+0000","last_active":"2024-03-07T03:11:49.478345+0000","last_peered":"2024-03-07T03:11:49.478345+0000","last_clean":"2024-03-07T03:11:49.478345+0000","last_became_active":"2024-03-07T03:11:17.758445+0000","last_became_peered":"2024-03-07T03:11:17.758445+0000","last_unstale":"2024-03-07T03:11:49.478345+0000","last_undegraded":"2024-03-07T03:11:49.478345+0000","last_fullsized":"2024-03-07T03:11:49.478345+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:09.182998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.475096+0000","last_change":"2024-03-07T03:11:17.725981+0000","last_active":"2024-03-07T03:11:49.475096+0000","last_peered":"2024-03-07T03:11:49.475096+0000","last_clean":"2024-03-07T03:11:49.475096+0000","last_became_active":"2024-03-07T03:11:17.722925+0000","last_became_peered":"2024-03-07T03:11:17.722925+0000","last_unstale":"2024-03-07T03:11:49.475096+0000","last_undegraded":"2024-03-07T03:11:49.475096+0000","last_fullsized":"2024-03-07T03:11:49.475096+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:19:53.431520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.869554+0000","last_change":"2024-03-07T03:11:19.031057+0000","last_active":"2024-03-07T03:11:49.869554+0000","last_peered":"2024-03-07T03:11:49.869554+0000","last_clean":"2024-03-07T03:11:49.869554+0000","last_became_active":"2024-03-07T03:11:19.030633+0000","last_became_peered":"2024-03-07T03:11:19.030633+0000","last_unstale":"2024-03-07T03:11:49.869554+0000","last_undegraded":"2024-03-07T03:11:49.869554+0000","last_fullsized":"2024-03-07T03:11:49.869554+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:44:01.959381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.485076+0000","last_change":"2024-03-07T03:11:17.740434+0000","last_active":"2024-03-07T03:11:49.485076+0000","last_peered":"2024-03-07T03:11:49.485076+0000","last_clean":"2024-03-07T03:11:49.485076+0000","last_became_active":"2024-03-07T03:11:17.739612+0000","last_became_peered":"2024-03-07T03:11:17.739612+0000","last_unstale":"2024-03-07T03:11:49.485076+0000","last_undegraded":"2024-03-07T03:11:49.485076+0000","last_fullsized":"2024-03-07T03:11:49.485076+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:55:39.231992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.478841+0000","last_change":"2024-03-07T03:11:17.762095+0000","last_active":"2024-03-07T03:11:49.478841+0000","last_peered":"2024-03-07T03:11:49.478841+0000","last_clean":"2024-03-07T03:11:49.478841+0000","last_became_active":"2024-03-07T03:11:17.761802+0000","last_became_peered":"2024-03-07T03:11:17.761802+0000","last_unstale":"2024-03-07T03:11:49.478841+0000","last_undegraded":"2024-03-07T03:11:49.478841+0000","last_fullsized":"2024-03-07T03:11:49.478841+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:40.710327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-07T03:11:49.480480+0000","last_change":"2024-03-07T03:11:17.733057+0000","last_active":"2024-03-07T03:11:49.480480+0000","last_peered":"2024-03-07T03:11:49.480480+0000","last_clean":"2024-03-07T03:11:49.480480+0000","last_became_active":"2024-03-07T03:11:17.732709+0000","last_became_peered":"2024-03-07T03:11:17.732709+0000","last_unstale":"2024-03-07T03:11:49.480480+0000","last_undegraded":"2024-03-07T03:11:49.480480+0000","last_fullsized":"2024-03-07T03:11:49.480480+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:14:18.485549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":77,"num_read_kb":80,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201343,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28352,"kb_used_data":1328,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716800,"statfs":{"total":95995035648,"available":95966003200,"internally_reserved":0,"allocated":1359872,"data_stored":916402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.92100000000000004,"15min":0.89500000000000002},"min":{"1min":0.57099999999999995,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.641,"5min":1.641,"15min":1.641},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.96399999999999997,"15min":0.96399999999999997},"min":{"1min":0.61099999999999999,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.2849999999999999,"5min":2.1909999999999998,"15min":2.1909999999999998},"last":1.202}]},{"osd":1,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.006,"15min":0.98899999999999999},"min":{"1min":0.72099999999999997,"5min":0.627,"15min":0.627},"max":{"1min":1.544,"5min":2.274,"15min":2.274},"last":1.0249999999999999},{"interface":"front","average":{"1min":0.89500000000000002,"5min":0.94299999999999995,"15min":0.96799999999999997},"min":{"1min":0.47599999999999998,"5min":0.439,"15min":0.439},"max":{"1min":1.125,"5min":1.536,"15min":1.536},"last":1.1519999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.0029999999999999,"15min":1.079},"min":{"1min":0.63600000000000001,"5min":0.502,"15min":0.502},"max":{"1min":1.7290000000000001,"5min":4.0270000000000001,"15min":4.0270000000000001},"last":0.78200000000000003},{"interface":"front","average":{"1min":1.103,"5min":1.04,"15min":1.04},"min":{"1min":0.63800000000000001,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.458,"5min":1.6719999999999999,"15min":1.6719999999999999},"last":1.2509999999999999}]},{"osd":3,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":1.0589999999999999,"15min":1.075},"min":{"1min":0.76200000000000001,"5min":0.66200000000000003,"15min":0.66200000000000003},"max":{"1min":1.383,"5min":2.0710000000000002,"15min":2.0710000000000002},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.0069999999999999,"15min":1.02},"min":{"1min":0.65200000000000002,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.143,"5min":1.4890000000000001,"15min":1.4890000000000001},"last":1.0900000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.98599999999999999,"15min":0.96799999999999997},"min":{"1min":0.70899999999999996,"5min":0.501,"15min":0.501},"max":{"1min":1.5860000000000001,"5min":1.831,"15min":1.831},"last":0.94499999999999995},{"interface":"front","average":{"1min":1.0069999999999999,"5min":0.99399999999999999,"15min":0.98199999999999998},"min":{"1min":0.52000000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.6819999999999999,"5min":1.8080000000000001,"15min":1.8080000000000001},"last":1.173}]},{"osd":5,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":1.0409999999999999,"15min":1.006},"min":{"1min":0.68100000000000005,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.5009999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.012,"15min":0.99099999999999999},"min":{"1min":0.59499999999999997,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.1970000000000001,"5min":2.1419999999999999,"15min":2.1419999999999999},"last":0.86699999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0800000000000001,"5min":1.0860000000000001,"15min":1.083},"min":{"1min":0.53400000000000003,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":1.474,"5min":2.0459999999999998,"15min":2.0459999999999998},"last":0.99299999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.0780000000000001,"15min":1.0449999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.7689999999999999,"5min":1.7689999999999999,"15min":1.7689999999999999},"last":1.0469999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462986,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.90800000000000003,"15min":0.85199999999999998},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.2430000000000001,"5min":1.409,"15min":1.409},"last":0.94399999999999995},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.90000000000000002,"15min":0.92600000000000005},"min":{"1min":0.53200000000000003,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":1.2629999999999999,"5min":1.363,"15min":1.363},"last":0.72699999999999998}]},{"osd":1,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.88200000000000001,"15min":0.91200000000000003},"min":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.1579999999999999,"5min":1.258,"15min":1.258},"last":1.054},{"interface":"front","average":{"1min":0.96499999999999997,"5min":0.94999999999999996,"15min":0.93300000000000005},"min":{"1min":0.51600000000000001,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":1.196,"5min":1.448,"15min":1.448},"last":0.80100000000000005}]},{"osd":2,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.95699999999999996,"15min":0.93999999999999995},"min":{"1min":0.68600000000000005,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.1970000000000001,"5min":1.4550000000000001,"15min":1.4550000000000001},"last":1.244},{"interface":"front","average":{"1min":0.92800000000000005,"5min":0.94999999999999996,"15min":0.95599999999999996},"min":{"1min":0.55500000000000005,"5min":0.37,"15min":0.37},"max":{"1min":1.1890000000000001,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":1.016}]},{"osd":3,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.93999999999999995,"15min":0.94799999999999995},"min":{"1min":0.53400000000000003,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":1.2110000000000001,"5min":1.3,"15min":1.3},"last":1.1259999999999999},{"interface":"front","average":{"1min":1.012,"5min":1.014,"15min":0.998},"min":{"1min":0.66100000000000003,"5min":0.56299999999999994,"15min":0.56299999999999994},"max":{"1min":1.28,"5min":1.536,"15min":1.536},"last":1.0940000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.90700000000000003,"15min":0.871},"min":{"1min":0.45700000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2749999999999999,"5min":1.488,"15min":1.488},"last":0.877},{"interface":"front","average":{"1min":0.93799999999999994,"5min":0.93200000000000005,"15min":0.91000000000000003},"min":{"1min":0.34899999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.282,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":0.90500000000000003}]},{"osd":5,"last update":"Thu Mar 7 03:12:09 2024","interfaces":[{"interface":"back","average":{"1min":1.0349999999999999,"5min":1.004,"15min":0.94699999999999995},"min":{"1min":0.42699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.3129999999999999,"5min":1.8759999999999999,"15min":1.8759999999999999},"last":1.1559999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.002,"15min":0.996},"min":{"1min":0.39800000000000002,"5min":0.375,"15min":0.375},"max":{"1min":1.2090000000000001,"5min":1.661,"15min":1.661},"last":0.98499999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.0649999999999999,"15min":1.0289999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.2829999999999999,"5min":1.994,"15min":1.994},"last":1.1899999999999999},{"interface":"front","average":{"1min":1.095,"5min":1.117,"15min":1.081},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":1.3240000000000001,"5min":1.8220000000000001,"15min":1.8220000000000001},"last":1.226}]}]},{"osd":1,"up_from":13,"seq":55834574971,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27744,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717408,"statfs":{"total":95995035648,"available":95966625792,"internally_reserved":0,"allocated":741376,"data_stored":324585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:10 2024","interfaces":[{"interface":"back","average":{"1min":1.173,"5min":0.79800000000000004,"15min":0.68200000000000005},"min":{"1min":0.374,"5min":0.29599999999999999,"15min":0.246},"max":{"1min":8.9090000000000007,"5min":8.9090000000000007,"15min":8.9090000000000007},"last":0.54100000000000004},{"interface":"front","average":{"1min":3.23,"5min":1.3100000000000001,"15min":0.84699999999999998},"min":{"1min":0.40100000000000002,"5min":0.39800000000000002,"15min":0.32100000000000001},"max":{"1min":45.719999999999999,"5min":45.719999999999999,"15min":45.719999999999999},"last":0.871}]},{"osd":2,"last update":"Thu Mar 7 03:12:54 2024","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.84299999999999997,"15min":0.71699999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.28599999999999998},"max":{"1min":1.1919999999999999,"5min":7.3170000000000002,"15min":7.3170000000000002},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.77600000000000002,"15min":0.68400000000000005},"min":{"1min":0.32700000000000001,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":1.8120000000000001,"5min":1.8120000000000001,"15min":1.8120000000000001},"last":1.135}]},{"osd":3,"last update":"Thu Mar 7 03:12:54 2024","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":0.98799999999999999,"15min":0.85299999999999998},"min":{"1min":0.40400000000000003,"5min":0.372,"15min":0.372},"max":{"1min":1.119,"5min":16.350000000000001,"15min":16.350000000000001},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.81899999999999995,"5min":0.99099999999999999,"15min":0.90000000000000002},"min":{"1min":0.38600000000000001,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.367,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":0.82299999999999995}]},{"osd":4,"last update":"Thu Mar 7 03:12:32 2024","interfaces":[{"interface":"back","average":{"1min":0.94499999999999995,"5min":1.1100000000000001,"15min":0.96999999999999997},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":1.2330000000000001,"5min":14.393000000000001,"15min":14.393000000000001},"last":1.2330000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.123,"15min":1.022},"min":{"1min":0.624,"5min":0.56399999999999995,"15min":0.56399999999999995},"max":{"1min":1.2450000000000001,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.94599999999999995}]},{"osd":5,"last update":"Thu Mar 7 03:12:24 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.1559999999999999,"15min":1.026},"min":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"max":{"1min":1.411,"5min":8.6560000000000006,"15min":8.6560000000000006},"last":1.1739999999999999},{"interface":"front","average":{"1min":1.046,"5min":1.1100000000000001,"15min":1.04},"min":{"1min":0.64300000000000002,"5min":0.624,"15min":0.624},"max":{"1min":1.54,"5min":7.1189999999999998,"15min":7.1189999999999998},"last":1.018}]},{"osd":6,"last update":"Thu Mar 7 03:12:10 2024","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":1.248,"15min":1.137},"min":{"1min":0.63300000000000001,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":16.262,"5min":16.262,"15min":16.262},"last":1.2050000000000001},{"interface":"front","average":{"1min":1.542,"5min":1.22,"15min":1.1419999999999999},"min":{"1min":0.69999999999999996,"5min":0.60599999999999998,"15min":0.60599999999999998},"max":{"1min":8.7910000000000004,"5min":8.7910000000000004,"15min":8.7910000000000004},"last":1.0429999999999999}]},{"osd":7,"last update":"Thu Mar 7 03:12:01 2024","interfaces":[{"interface":"back","average":{"1min":1.131,"5min":1.0980000000000001,"15min":1.0840000000000001},"min":{"1min":0.81100000000000005,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.8580000000000001,"5min":1.8580000000000001,"15min":1.8580000000000001},"last":1.29},{"interface":"front","average":{"1min":1.5800000000000001,"5min":1.264,"15min":1.218},"min":{"1min":0.67700000000000005,"5min":0.67400000000000004,"15min":0.67400000000000004},"max":{"1min":8.0909999999999993,"5min":8.0909999999999993,"15min":8.0909999999999993},"last":1.0669999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738501,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Thu Mar 7 03:12:13 2024","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.70799999999999996,"15min":0.60599999999999998},"min":{"1min":0.40999999999999998,"5min":0.253,"15min":0.246},"max":{"1min":0.98299999999999998,"5min":1.494,"15min":1.494},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.74199999999999999,"5min":0.74099999999999999,"15min":0.64800000000000002},"min":{"1min":0.34200000000000003,"5min":0.314,"15min":0.23200000000000001},"max":{"1min":1.0720000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.0820000000000001}]},{"osd":2,"last update":"Thu Mar 7 03:12:03 2024","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":0.84099999999999997,"15min":0.72599999999999998},"min":{"1min":0.43099999999999999,"5min":0.35599999999999998,"15min":0.29099999999999998},"max":{"1min":1.337,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.1639999999999999},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.76500000000000001,"15min":0.70499999999999996},"min":{"1min":0.48099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.032,"5min":1.0940000000000001,"15min":1.0940000000000001},"last":0.76300000000000001}]},{"osd":3,"last update":"Thu Mar 7 03:12:48 2024","interfaces":[{"interface":"back","average":{"1min":0.89000000000000001,"5min":0.90700000000000003,"15min":0.82599999999999996},"min":{"1min":0.41599999999999998,"5min":0.376,"15min":0.376},"max":{"1min":1.246,"5min":1.788,"15min":1.788},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.78600000000000003,"5min":0.84199999999999997,"15min":0.81100000000000005},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.177,"5min":1.5820000000000001,"15min":1.5820000000000001},"last":1.1200000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:12:35 2024","interfaces":[{"interface":"back","average":{"1min":0.871,"5min":0.93400000000000005,"15min":0.878},"min":{"1min":0.626,"5min":0.60399999999999998,"15min":0.59199999999999997},"max":{"1min":1.137,"5min":1.5409999999999999,"15min":1.5409999999999999},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.085,"15min":0.98999999999999999},"min":{"1min":0.75600000000000001,"5min":0.61599999999999999,"15min":0.61599999999999999},"max":{"1min":1.405,"5min":1.46,"15min":1.46},"last":1.3129999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:12:29 2024","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":1.0960000000000001,"15min":1.042},"min":{"1min":0.77800000000000002,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.486,"5min":1.6639999999999999,"15min":1.6639999999999999},"last":1.2430000000000001},{"interface":"front","average":{"1min":1.123,"5min":1.087,"15min":1.0169999999999999},"min":{"1min":0.66900000000000004,"5min":0.63200000000000001,"15min":0.63200000000000001},"max":{"1min":1.4159999999999999,"5min":1.651,"15min":1.651},"last":0.90800000000000003}]},{"osd":6,"last update":"Thu Mar 7 03:12:13 2024","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":1.1200000000000001,"15min":1.1319999999999999},"min":{"1min":0.64600000000000002,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.4830000000000001,"5min":1.7290000000000001,"15min":1.7290000000000001},"last":0.995},{"interface":"front","average":{"1min":1.141,"5min":1.125,"15min":1.1120000000000001},"min":{"1min":0.70999999999999996,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.514,"5min":1.7070000000000001,"15min":1.7070000000000001},"last":1.032}]},{"osd":7,"last update":"Thu Mar 7 03:12:03 2024","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":1.0660000000000001,"15min":1.087},"min":{"1min":0.68200000000000005,"5min":0.625,"15min":0.625},"max":{"1min":1.4750000000000001,"5min":1.5920000000000001,"15min":1.5920000000000001},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.228,"5min":1.208,"15min":1.238},"min":{"1min":0.71799999999999997,"5min":0.65500000000000003,"15min":0.65500000000000003},"max":{"1min":1.5620000000000001,"5min":1.73,"15min":1.73},"last":1.389}]}]},{"osd":2,"up_from":18,"seq":77309411441,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":749568,"data_stored":325201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.70199999999999996,"15min":0.63900000000000001},"min":{"1min":0.437,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.1180000000000001,"5min":1.1459999999999999,"15min":1.2929999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.75800000000000001,"15min":0.65900000000000003},"min":{"1min":0.39400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.2,"5min":1.2,"15min":1.2},"last":0.63600000000000001}]},{"osd":1,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":0.75800000000000001,"15min":0.68500000000000005},"min":{"1min":0.36399999999999999,"5min":0.34200000000000003,"15min":0.33200000000000002},"max":{"1min":1.167,"5min":1.218,"15min":1.218},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.88300000000000001,"5min":0.82499999999999996,"15min":0.72799999999999998},"min":{"1min":0.46100000000000002,"5min":0.373,"15min":0.312},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.89300000000000002}]},{"osd":3,"last update":"Thu Mar 7 03:12:48 2024","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":0.78500000000000003,"15min":0.79400000000000004},"min":{"1min":0.379,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.999,"5min":1.2150000000000001,"15min":1.2150000000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.84699999999999998,"15min":0.84099999999999997},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.0860000000000001,"5min":1.4079999999999999,"15min":1.4079999999999999},"last":0.78900000000000003}]},{"osd":4,"last update":"Thu Mar 7 03:12:35 2024","interfaces":[{"interface":"back","average":{"1min":1.02,"5min":1.0109999999999999,"15min":0.99099999999999999},"min":{"1min":0.70199999999999996,"5min":0.59799999999999998,"15min":0.59799999999999998},"max":{"1min":1.383,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.997},{"interface":"front","average":{"1min":0.88100000000000001,"5min":0.91000000000000003,"15min":0.91800000000000004},"min":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.147,"5min":1.2789999999999999,"15min":1.2789999999999999},"last":0.95999999999999996}]},{"osd":5,"last update":"Thu Mar 7 03:12:17 2024","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.03,"15min":1.0309999999999999},"min":{"1min":0.66900000000000004,"5min":0.628,"15min":0.628},"max":{"1min":1.325,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.0680000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.93799999999999994,"15min":0.96599999999999997},"min":{"1min":0.60299999999999998,"5min":0.58599999999999997,"15min":0.58599999999999997},"max":{"1min":1.2290000000000001,"5min":1.25,"15min":1.25},"last":1.224}]},{"osd":6,"last update":"Thu Mar 7 03:12:12 2024","interfaces":[{"interface":"back","average":{"1min":1.091,"5min":1.056,"15min":1.1080000000000001},"min":{"1min":0.59399999999999997,"5min":0.57099999999999995,"15min":0.57099999999999995},"max":{"1min":1.353,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":0.92600000000000005},{"interface":"front","average":{"1min":1.1319999999999999,"5min":1.109,"15min":1.139},"min":{"1min":0.64800000000000002,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.5189999999999999,"5min":1.5700000000000001,"15min":1.5700000000000001},"last":0.94499999999999995}]},{"osd":7,"last update":"Thu Mar 7 03:12:05 2024","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.0369999999999999,"15min":1.0449999999999999},"min":{"1min":0.82399999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.387,"5min":1.5640000000000001,"15min":1.5640000000000001},"last":1.131},{"interface":"front","average":{"1min":1.179,"5min":1.123,"15min":1.139},"min":{"1min":0.73399999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"max":{"1min":1.361,"5min":1.611,"15min":1.611},"last":1.0189999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149800,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27744,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717408,"statfs":{"total":95995035648,"available":95966625792,"internally_reserved":0,"allocated":745472,"data_stored":324675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:45 2024","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.38300000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.2509999999999999,"5min":1.2509999999999999,"15min":1.6419999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.745,"15min":0.73099999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.1180000000000001,"5min":1.321,"15min":1.321},"last":0.76900000000000002}]},{"osd":1,"last update":"Thu Mar 7 03:12:45 2024","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.76100000000000001,"15min":0.78800000000000003},"min":{"1min":0.34499999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":1.196,"5min":1.2709999999999999,"15min":1.4550000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.72999999999999998,"15min":0.74199999999999999},"min":{"1min":0.316,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.25,"5min":1.25,"15min":1.25},"last":1.177}]},{"osd":2,"last update":"Thu Mar 7 03:12:45 2024","interfaces":[{"interface":"back","average":{"1min":0.81799999999999995,"5min":0.80300000000000005,"15min":0.81399999999999995},"min":{"1min":0.36599999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.5109999999999999},"last":0.98399999999999999},{"interface":"front","average":{"1min":0.73099999999999998,"5min":0.73999999999999999,"15min":0.77700000000000002},"min":{"1min":0.34899999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.087,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.91200000000000003}]},{"osd":4,"last update":"Thu Mar 7 03:12:28 2024","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":0.96999999999999997,"15min":0.93500000000000005},"min":{"1min":0.71599999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.292,"5min":1.5289999999999999,"15min":1.5289999999999999},"last":0.85499999999999998},{"interface":"front","average":{"1min":1.02,"5min":1.012,"15min":0.97799999999999998},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.4530000000000001,"5min":1.4530000000000001,"15min":1.4530000000000001},"last":1.1339999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:12:28 2024","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.92500000000000004,"15min":0.95299999999999996},"min":{"1min":0.71499999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"max":{"1min":1.224,"5min":1.2490000000000001,"15min":1.268},"last":1.343},{"interface":"front","average":{"1min":1.0349999999999999,"5min":1.0409999999999999,"15min":1.0569999999999999},"min":{"1min":0.67000000000000004,"5min":0.53200000000000003,"15min":0.53200000000000003},"max":{"1min":1.5940000000000001,"5min":1.5940000000000001,"15min":1.5940000000000001},"last":0.95299999999999996}]},{"osd":6,"last update":"Thu Mar 7 03:12:06 2024","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.046,"15min":1.0960000000000001},"min":{"1min":0.74099999999999999,"5min":0.60399999999999998,"15min":0.60399999999999998},"max":{"1min":1.55,"5min":1.55,"15min":1.55},"last":0.81499999999999995},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.98999999999999999,"15min":1.034},"min":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"max":{"1min":1.3520000000000001,"5min":1.5840000000000001,"15min":1.5840000000000001},"last":1.397}]},{"osd":7,"last update":"Thu Mar 7 03:12:02 2024","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.121,"15min":1.1850000000000001},"min":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.5189999999999999,"5min":1.6120000000000001,"15min":1.6120000000000001},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.077,"5min":1.139,"15min":1.2050000000000001},"min":{"1min":0.626,"5min":0.626,"15min":0.626},"max":{"1min":1.419,"5min":1.5669999999999999,"15min":1.5669999999999999},"last":1.081}]}]},{"osd":4,"up_from":31,"seq":133143986269,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27768,"kb_used_data":744,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717384,"statfs":{"total":95995035648,"available":95966601216,"internally_reserved":0,"allocated":761856,"data_stored":326022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":0.88700000000000001,"15min":0.81299999999999994},"min":{"1min":0.64800000000000002,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.91900000000000004},{"interface":"front","average":{"1min":0.92300000000000004,"5min":0.91100000000000003,"15min":0.84699999999999998},"min":{"1min":0.70799999999999996,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.2809999999999999,"5min":1.2809999999999999,"15min":1.2809999999999999},"last":1.0409999999999999}]},{"osd":1,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":0.91700000000000004,"15min":0.86699999999999999},"min":{"1min":0.627,"5min":0.497,"15min":0.497},"max":{"1min":1.206,"5min":1.343,"15min":1.343},"last":1.2549999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.93500000000000005,"15min":0.86499999999999999},"min":{"1min":0.64900000000000002,"5min":0.54300000000000004,"15min":0.54300000000000004},"max":{"1min":1.3360000000000001,"5min":1.3360000000000001,"15min":1.3360000000000001},"last":0.85099999999999998}]},{"osd":2,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":0.96199999999999997,"15min":0.90800000000000003},"min":{"1min":0.64600000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":2.024,"5min":2.024,"15min":2.024},"last":1.02},{"interface":"front","average":{"1min":1.0640000000000001,"5min":0.99199999999999999,"15min":0.89900000000000002},"min":{"1min":0.72899999999999998,"5min":0.54700000000000004,"15min":0.54700000000000004},"max":{"1min":1.4379999999999999,"5min":1.4379999999999999,"15min":1.4379999999999999},"last":0.97599999999999998}]},{"osd":3,"last update":"Thu Mar 7 03:12:30 2024","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.96499999999999997,"15min":0.873},"min":{"1min":0.77300000000000002,"5min":0.53500000000000003,"15min":0.53500000000000003},"max":{"1min":1.851,"5min":1.851,"15min":1.851},"last":0.995},{"interface":"front","average":{"1min":0.98799999999999999,"5min":1.0089999999999999,"15min":0.93200000000000005},"min":{"1min":0.70599999999999996,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.3939999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":1.1759999999999999}]},{"osd":5,"last update":"Thu Mar 7 03:12:23 2024","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":0.92200000000000004,"15min":0.78300000000000003},"min":{"1min":0.69699999999999995,"5min":0.41199999999999998,"15min":0.371},"max":{"1min":1.3440000000000001,"5min":1.369,"15min":1.369},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":0.91900000000000004,"15min":0.82699999999999996},"min":{"1min":0.64500000000000002,"5min":0.48399999999999999,"15min":0.34999999999999998},"max":{"1min":1.127,"5min":1.552,"15min":1.552},"last":0.81699999999999995}]},{"osd":6,"last update":"Thu Mar 7 03:12:11 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.0069999999999999,"15min":0.92000000000000004},"min":{"1min":0.68300000000000005,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.218,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":1.071},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.0469999999999999,"15min":0.92900000000000005},"min":{"1min":0.73199999999999998,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.4450000000000001,"5min":1.754,"15min":1.754},"last":0.747}]},{"osd":7,"last update":"Thu Mar 7 03:11:56 2024","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.0529999999999999,"15min":1.0169999999999999},"min":{"1min":0.59299999999999997,"5min":0.436,"15min":0.436},"max":{"1min":2.2189999999999999,"5min":2.2189999999999999,"15min":2.2189999999999999},"last":0.94099999999999995},{"interface":"front","average":{"1min":1.127,"5min":1.149,"15min":1.119},"min":{"1min":0.60999999999999999,"5min":0.53700000000000003,"15min":0.53700000000000003},"max":{"1min":2.0760000000000001,"5min":2.0760000000000001,"15min":2.0760000000000001},"last":1.3320000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724628,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":745472,"data_stored":324906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":27654411},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.90200000000000002,"15min":0.89900000000000002},"min":{"1min":0.60299999999999998,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.236,"5min":1.597,"15min":1.597},"last":0.878},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.93400000000000005,"15min":0.87},"min":{"1min":0.68999999999999995,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.2749999999999999,"5min":1.3,"15min":1.3},"last":1.24}]},{"osd":1,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.95799999999999996,"15min":0.90700000000000003},"min":{"1min":0.67800000000000005,"5min":0.629,"15min":0.51700000000000002},"max":{"1min":1.2609999999999999,"5min":1.3779999999999999,"15min":1.3779999999999999},"last":1.2090000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.92800000000000005,"15min":0.89800000000000002},"min":{"1min":0.65200000000000002,"5min":0.501,"15min":0.501},"max":{"1min":1.23,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":1.1339999999999999}]},{"osd":2,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.92500000000000004,"15min":0.88400000000000001},"min":{"1min":0.64700000000000002,"5min":0.47999999999999998,"15min":0.47999999999999998},"max":{"1min":1.4259999999999999,"5min":1.4259999999999999,"15min":1.4259999999999999},"last":0.97899999999999998},{"interface":"front","average":{"1min":0.97899999999999998,"5min":0.97499999999999998,"15min":0.93000000000000005},"min":{"1min":0.69799999999999995,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.1719999999999999,"5min":1.407,"15min":1.407},"last":1.335}]},{"osd":3,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":0.96499999999999997,"15min":0.93100000000000005},"min":{"1min":0.80600000000000005,"5min":0.497,"15min":0.497},"max":{"1min":1.546,"5min":1.546,"15min":1.546},"last":1.163},{"interface":"front","average":{"1min":1.071,"5min":1.0249999999999999,"15min":0.99399999999999999},"min":{"1min":0.77900000000000003,"5min":0.51300000000000001,"15min":0.51300000000000001},"max":{"1min":1.337,"5min":1.6200000000000001,"15min":1.6200000000000001},"last":1.1100000000000001}]},{"osd":4,"last update":"Thu Mar 7 03:12:25 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.90500000000000003,"15min":0.78100000000000003},"min":{"1min":0.69699999999999995,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":1.2370000000000001,"5min":1.571,"15min":1.571},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.085,"5min":0.95799999999999996,"15min":0.84699999999999998},"min":{"1min":0.81899999999999995,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.6459999999999999,"5min":1.6459999999999999,"15min":1.6459999999999999},"last":1.2749999999999999}]},{"osd":6,"last update":"Thu Mar 7 03:12:15 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":0.99299999999999999,"15min":0.879},"min":{"1min":0.74299999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":1.2509999999999999,"5min":1.911,"15min":1.911},"last":0.93899999999999995},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.0089999999999999,"15min":0.89900000000000002},"min":{"1min":0.78800000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.74,"5min":1.74,"15min":1.74},"last":1.3560000000000001}]},{"osd":7,"last update":"Thu Mar 7 03:12:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":1.0629999999999999,"15min":1.036},"min":{"1min":0.53800000000000003,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3720000000000001,"5min":1.7,"15min":1.7},"last":1.0349999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.109,"15min":1.0860000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":1.3280000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"last":1.306}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:12:56.693 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-07T03:12:56.693 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-07T03:12:56.693 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-07T03:12:56.693 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:12:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:56 smithi148 bash[21362]: cluster 2024-03-07T03:12:55.048340+0000 mgr.y (mgr.24287) 164 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:57.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:56 smithi148 bash[31149]: cluster 2024-03-07T03:12:55.048340+0000 mgr.y (mgr.24287) 164 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:57.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:56 smithi195 bash[23221]: cluster 2024-03-07T03:12:55.048340+0000 mgr.y (mgr.24287) 164 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:57 smithi148 bash[21362]: audit 2024-03-07T03:12:56.094419+0000 mgr.y (mgr.24287) 165 : audit 0 from='client.14718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:12:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:57 smithi148 bash[31149]: audit 2024-03-07T03:12:56.094419+0000 mgr.y (mgr.24287) 165 : audit 0 from='client.14718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:12:58.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:57 smithi195 bash[23221]: audit 2024-03-07T03:12:56.094419+0000 mgr.y (mgr.24287) 165 : audit 0 from='client.14718 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:12:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:12:58 smithi148 bash[21362]: cluster 2024-03-07T03:12:57.048975+0000 mgr.y (mgr.24287) 166 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:12:58 smithi148 bash[31149]: cluster 2024-03-07T03:12:57.048975+0000 mgr.y (mgr.24287) 166 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:12:59.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:12:58 smithi195 bash[23221]: cluster 2024-03-07T03:12:57.048975+0000 mgr.y (mgr.24287) 166 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:01.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:00 smithi148 bash[21362]: cluster 2024-03-07T03:12:59.049618+0000 mgr.y (mgr.24287) 167 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:01.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:00 smithi148 bash[31149]: cluster 2024-03-07T03:12:59.049618+0000 mgr.y (mgr.24287) 167 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:01.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:00 smithi195 bash[23221]: cluster 2024-03-07T03:12:59.049618+0000 mgr.y (mgr.24287) 167 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:01.419 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:13:02.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:02 smithi148 bash[21362]: cluster 2024-03-07T03:13:01.050706+0000 mgr.y (mgr.24287) 168 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:02.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:02 smithi148 bash[31149]: cluster 2024-03-07T03:13:01.050706+0000 mgr.y (mgr.24287) 168 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:03.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:02 smithi195 bash[23221]: cluster 2024-03-07T03:13:01.050706+0000 mgr.y (mgr.24287) 168 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:04.345 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:13:04.346 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-07T03:13:04.953 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:04 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:13:04] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2024-03-07T03:13:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:04 smithi148 bash[31149]: cluster 2024-03-07T03:13:03.051320+0000 mgr.y (mgr.24287) 169 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:04 smithi148 bash[31149]: audit 2024-03-07T03:13:04.047849+0000 mon.b (mon.1) 118 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:04 smithi148 bash[31149]: audit 2024-03-07T03:13:04.359178+0000 mon.a (mon.0) 846 : audit 0 from='client.? 172.21.15.148:0/3950298579' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:13:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:04 smithi148 bash[21362]: cluster 2024-03-07T03:13:03.051320+0000 mgr.y (mgr.24287) 169 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:04 smithi148 bash[21362]: audit 2024-03-07T03:13:04.047849+0000 mon.b (mon.1) 118 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:04 smithi148 bash[21362]: audit 2024-03-07T03:13:04.359178+0000 mon.a (mon.0) 846 : audit 0 from='client.? 172.21.15.148:0/3950298579' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:13:04.954 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-07T03:13:04.955 INFO:teuthology.run_tasks:Running task print... 2024-03-07T03:13:04.964 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-03-07T03:13:04.964 INFO:teuthology.run_tasks:Running task parallel... 2024-03-07T03:13:04.972 INFO:teuthology.task.parallel:starting parallel... 2024-03-07T03:13:04.972 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-07T03:13:04.972 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-03-07T03:13:04.977 INFO:tasks.workunit:Pulling workunits from ref reef 2024-03-07T03:13:04.977 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-03-07T03:13:04.977 DEBUG:teuthology.orchestra.run.smithi148:> stat -- /home/ubuntu/cephtest/mnt.0 2024-03-07T03:13:04.978 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-07T03:13:04.978 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-07T03:13:04.978 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi148.front.sepia.ceph.com 2024-03-07T03:13:04.978 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-03-07T03:13:04.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T03:13:04.985 INFO:teuthology.orchestra.run.smithi148.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-03-07T03:13:04.985 DEBUG:teuthology.orchestra.run.smithi148:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-03-07T03:13:05.031 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-03-07T03:13:05.031 DEBUG:teuthology.orchestra.run.smithi148:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-03-07T03:13:05.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:04 smithi195 bash[23221]: cluster 2024-03-07T03:13:03.051320+0000 mgr.y (mgr.24287) 169 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:05.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:04 smithi195 bash[23221]: audit 2024-03-07T03:13:04.047849+0000 mon.b (mon.1) 118 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:05.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:04 smithi195 bash[23221]: audit 2024-03-07T03:13:04.359178+0000 mon.a (mon.0) 846 : audit 0 from='client.? 172.21.15.148:0/3950298579' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:13:05.079 INFO:tasks.workunit:timeout=3h 2024-03-07T03:13:05.079 INFO:tasks.workunit:cleanup=True 2024-03-07T03:13:05.080 DEBUG:teuthology.orchestra.run.smithi148:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-03-07T03:13:05.126 INFO:tasks.workunit.client.0.smithi148.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-03-07T03:13:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:06 smithi148 bash[21362]: cluster 2024-03-07T03:13:05.052053+0000 mgr.y (mgr.24287) 170 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:07.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:06 smithi148 bash[31149]: cluster 2024-03-07T03:13:05.052053+0000 mgr.y (mgr.24287) 170 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:07.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:06 smithi195 bash[23221]: cluster 2024-03-07T03:13:05.052053+0000 mgr.y (mgr.24287) 170 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:09.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:08 smithi148 bash[21362]: cluster 2024-03-07T03:13:07.052675+0000 mgr.y (mgr.24287) 171 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:09.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:08 smithi148 bash[31149]: cluster 2024-03-07T03:13:07.052675+0000 mgr.y (mgr.24287) 171 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:09.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:08 smithi195 bash[23221]: cluster 2024-03-07T03:13:07.052675+0000 mgr.y (mgr.24287) 171 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:09.476 DEBUG:teuthology.orchestra.run.smithi148:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-03-07T03:13:09.522 INFO:tasks.workunit.client.0.smithi148.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-03-07T03:13:09.526 INFO:tasks.workunit.client.0.smithi148.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-07T03:13:09.526 INFO:tasks.workunit.client.0.smithi148.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-03-07T03:13:09.714 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:13:11.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:10 smithi148 bash[21362]: cluster 2024-03-07T03:13:09.053401+0000 mgr.y (mgr.24287) 172 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:11.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:10 smithi148 bash[31149]: cluster 2024-03-07T03:13:09.053401+0000 mgr.y (mgr.24287) 172 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:11.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:10 smithi195 bash[23221]: cluster 2024-03-07T03:13:09.053401+0000 mgr.y (mgr.24287) 172 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:13.012 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:12 smithi148 bash[21362]: cluster 2024-03-07T03:13:11.054687+0000 mgr.y (mgr.24287) 173 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:13.012 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:12 smithi148 bash[31149]: cluster 2024-03-07T03:13:11.054687+0000 mgr.y (mgr.24287) 173 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:13.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:12 smithi195 bash[23221]: cluster 2024-03-07T03:13:11.054687+0000 mgr.y (mgr.24287) 173 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:13.658 INFO:tasks.workunit.client.0.smithi148.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-03-07T03:13:14.001 INFO:tasks.workunit.client.0.smithi148.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-03-07T03:13:14.117 INFO:tasks.workunit.client.0.smithi148.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-07T03:13:14.119 INFO:tasks.workunit.client.0.smithi148.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-07T03:13:14.119 INFO:tasks.workunit.client.0.smithi148.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-03-07T03:13:14.168 INFO:tasks.workunit.client.0.smithi148.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-07T03:13:14.172 DEBUG:teuthology.orchestra.run.smithi148:> set -ex 2024-03-07T03:13:14.172 DEBUG:teuthology.orchestra.run.smithi148:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-03-07T03:13:14.218 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-03-07T03:13:14.219 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-03-07T03:13:14.219 DEBUG:teuthology.orchestra.run.smithi148:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-03-07T03:13:14.423 INFO:tasks.workunit.client.0.smithi148.stderr:warning: failed to enable lockdep 2024-03-07T03:13:14.423 INFO:tasks.workunit.client.0.smithi148.stdout:seed 68562 2024-03-07T03:13:14.423 INFO:tasks.workunit.client.0.smithi148.stdout:[==========] Running 476 tests from 24 test suites. 2024-03-07T03:13:14.423 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] Global test environment set-up. 2024-03-07T03:13:14.423 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 1 test from TestJournalStress 2024-03-07T03:13:14.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:14 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:13:14] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:14 smithi148 bash[21362]: cluster 2024-03-07T03:13:13.055353+0000 mgr.y (mgr.24287) 174 : cluster 0 pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:14 smithi148 bash[31149]: cluster 2024-03-07T03:13:13.055353+0000 mgr.y (mgr.24287) 174 : cluster 0 pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:14 smithi148 bash[21615]: [07/Mar/2024:03:13:14] ENGINE Bus STOPPING 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:14 smithi148 bash[21615]: [07/Mar/2024:03:13:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:14 smithi148 bash[21615]: [07/Mar/2024:03:13:14] ENGINE Bus STOPPED 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:14 smithi148 bash[21615]: [07/Mar/2024:03:13:14] ENGINE Bus STARTING 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[21615]: [07/Mar/2024:03:13:15] ENGINE Serving on http://:::9283 2024-03-07T03:13:15.290 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[21615]: [07/Mar/2024:03:13:15] ENGINE Bus STARTED 2024-03-07T03:13:15.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:14 smithi195 bash[23221]: cluster 2024-03-07T03:13:13.055353+0000 mgr.y (mgr.24287) 174 : cluster 0 pgmap v147: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:15.692 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-07T03:13:15.955 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[31149]: audit 2024-03-07T03:13:14.913197+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[31149]: audit 2024-03-07T03:13:14.945587+0000 mon.b (mon.1) 119 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[31149]: cluster 2024-03-07T03:13:14.956351+0000 mon.a (mon.0) 848 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[31149]: audit 2024-03-07T03:13:14.957368+0000 mon.a (mon.0) 849 : audit 1 from='client.? 172.21.15.148:0/68931354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[21362]: audit 2024-03-07T03:13:14.913197+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[21362]: audit 2024-03-07T03:13:14.945587+0000 mon.b (mon.1) 119 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[21362]: cluster 2024-03-07T03:13:14.956351+0000 mon.a (mon.0) 848 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-03-07T03:13:16.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:15 smithi148 bash[21362]: audit 2024-03-07T03:13:14.957368+0000 mon.a (mon.0) 849 : audit 1 from='client.? 172.21.15.148:0/68931354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:16.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:15 smithi195 bash[23221]: audit 2024-03-07T03:13:14.913197+0000 mon.a (mon.0) 847 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:13:16.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:15 smithi195 bash[23221]: audit 2024-03-07T03:13:14.945587+0000 mon.b (mon.1) 119 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:16.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:15 smithi195 bash[23221]: cluster 2024-03-07T03:13:14.956351+0000 mon.a (mon.0) 848 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-03-07T03:13:16.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:15 smithi195 bash[23221]: audit 2024-03-07T03:13:14.957368+0000 mon.a (mon.0) 849 : audit 1 from='client.? 172.21.15.148:0/68931354' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:17.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[21362]: cluster 2024-03-07T03:13:15.056024+0000 mgr.y (mgr.24287) 175 : cluster 0 pgmap v149: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:17.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[21362]: cluster 2024-03-07T03:13:15.954449+0000 mon.a (mon.0) 850 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:17.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[21362]: audit 2024-03-07T03:13:15.963521+0000 mon.a (mon.0) 851 : audit 1 from='client.? 172.21.15.148:0/68931354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:17.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[21362]: cluster 2024-03-07T03:13:15.964887+0000 mon.a (mon.0) 852 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-03-07T03:13:17.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[31149]: cluster 2024-03-07T03:13:15.056024+0000 mgr.y (mgr.24287) 175 : cluster 0 pgmap v149: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:17.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[31149]: cluster 2024-03-07T03:13:15.954449+0000 mon.a (mon.0) 850 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:17.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[31149]: audit 2024-03-07T03:13:15.963521+0000 mon.a (mon.0) 851 : audit 1 from='client.? 172.21.15.148:0/68931354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:17.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:16 smithi148 bash[31149]: cluster 2024-03-07T03:13:15.964887+0000 mon.a (mon.0) 852 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-03-07T03:13:17.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:16 smithi195 bash[23221]: cluster 2024-03-07T03:13:15.056024+0000 mgr.y (mgr.24287) 175 : cluster 0 pgmap v149: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:17.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:16 smithi195 bash[23221]: cluster 2024-03-07T03:13:15.954449+0000 mon.a (mon.0) 850 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:17.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:16 smithi195 bash[23221]: audit 2024-03-07T03:13:15.963521+0000 mon.a (mon.0) 851 : audit 1 from='client.? 172.21.15.148:0/68931354' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:17.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:16 smithi195 bash[23221]: cluster 2024-03-07T03:13:15.964887+0000 mon.a (mon.0) 852 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-03-07T03:13:18.000 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Stress.cc:37: Skipped 2024-03-07T03:13:18.000 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:18.000 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalStress.DiscardWithPruneWriteOverlap (2045 ms) 2024-03-07T03:13:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:17 smithi148 bash[21362]: cluster 2024-03-07T03:13:16.967265+0000 mon.a (mon.0) 853 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-03-07T03:13:18.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:17 smithi148 bash[31149]: cluster 2024-03-07T03:13:16.967265+0000 mon.a (mon.0) 853 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-03-07T03:13:18.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:17 smithi195 bash[23221]: cluster 2024-03-07T03:13:16.967265+0000 mon.a (mon.0) 853 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-03-07T03:13:18.979 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 1 test from TestJournalStress (2045 ms total) 2024-03-07T03:13:18.980 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:13:18.980 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 16 tests from TestJournalReplay 2024-03-07T03:13:19.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[21362]: cluster 2024-03-07T03:13:17.056642+0000 mgr.y (mgr.24287) 176 : cluster 0 pgmap v152: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:19.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[21362]: cluster 2024-03-07T03:13:17.960374+0000 mon.a (mon.0) 854 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:19.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[21362]: cluster 2024-03-07T03:13:17.960446+0000 mon.a (mon.0) 855 : cluster 1 Cluster is now healthy 2024-03-07T03:13:19.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[21362]: cluster 2024-03-07T03:13:17.977512+0000 mon.a (mon.0) 856 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-03-07T03:13:19.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[31149]: cluster 2024-03-07T03:13:17.056642+0000 mgr.y (mgr.24287) 176 : cluster 0 pgmap v152: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:19.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[31149]: cluster 2024-03-07T03:13:17.960374+0000 mon.a (mon.0) 854 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:19.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[31149]: cluster 2024-03-07T03:13:17.960446+0000 mon.a (mon.0) 855 : cluster 1 Cluster is now healthy 2024-03-07T03:13:19.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:18 smithi148 bash[31149]: cluster 2024-03-07T03:13:17.977512+0000 mon.a (mon.0) 856 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-03-07T03:13:19.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:18 smithi195 bash[23221]: cluster 2024-03-07T03:13:17.056642+0000 mgr.y (mgr.24287) 176 : cluster 0 pgmap v152: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:19.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:18 smithi195 bash[23221]: cluster 2024-03-07T03:13:17.960374+0000 mon.a (mon.0) 854 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:19.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:18 smithi195 bash[23221]: cluster 2024-03-07T03:13:17.960446+0000 mon.a (mon.0) 855 : cluster 1 Cluster is now healthy 2024-03-07T03:13:19.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:18 smithi195 bash[23221]: cluster 2024-03-07T03:13:17.977512+0000 mon.a (mon.0) 856 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-03-07T03:13:19.802 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:13:20.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:19 smithi148 bash[21362]: cluster 2024-03-07T03:13:18.987289+0000 mon.a (mon.0) 857 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-03-07T03:13:20.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:19 smithi148 bash[21362]: audit 2024-03-07T03:13:19.048014+0000 mon.b (mon.1) 120 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:20.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:19 smithi148 bash[31149]: cluster 2024-03-07T03:13:18.987289+0000 mon.a (mon.0) 857 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-03-07T03:13:20.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:19 smithi148 bash[31149]: audit 2024-03-07T03:13:19.048014+0000 mon.b (mon.1) 120 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:20.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:19 smithi195 bash[23221]: cluster 2024-03-07T03:13:18.987289+0000 mon.a (mon.0) 857 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-03-07T03:13:20.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:19 smithi195 bash[23221]: audit 2024-03-07T03:13:19.048014+0000 mon.b (mon.1) 120 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:20.991 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-03-07T03:13:21.075 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: cluster 2024-03-07T03:13:19.056992+0000 mgr.y (mgr.24287) 177 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:21.075 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: cluster 2024-03-07T03:13:19.994121+0000 mon.a (mon.0) 858 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-03-07T03:13:21.075 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:19.995108+0000 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.148:0/2618491298' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:21.075 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:20.392908+0000 mon.a (mon.0) 860 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.075 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:20.404805+0000 mon.a (mon.0) 861 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:20.700807+0000 mon.a (mon.0) 862 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:20.716941+0000 mon.a (mon.0) 863 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:20.718525+0000 mon.b (mon.1) 121 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:21.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:20.720752+0000 mon.b (mon.1) 122 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:21.076 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[31149]: audit 2024-03-07T03:13:20.735669+0000 mon.a (mon.0) 864 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.077 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: cluster 2024-03-07T03:13:19.056992+0000 mgr.y (mgr.24287) 177 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:21.077 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: cluster 2024-03-07T03:13:19.994121+0000 mon.a (mon.0) 858 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-03-07T03:13:21.077 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:19.995108+0000 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.148:0/2618491298' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:21.077 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:20.392908+0000 mon.a (mon.0) 860 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.077 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:20.404805+0000 mon.a (mon.0) 861 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.077 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:20.700807+0000 mon.a (mon.0) 862 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.078 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:20.716941+0000 mon.a (mon.0) 863 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.078 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:20.718525+0000 mon.b (mon.1) 121 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:21.078 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:20.720752+0000 mon.b (mon.1) 122 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:21.078 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:21 smithi148 bash[21362]: audit 2024-03-07T03:13:20.735669+0000 mon.a (mon.0) 864 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: cluster 2024-03-07T03:13:19.056992+0000 mgr.y (mgr.24287) 177 : cluster 0 pgmap v155: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:21.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: cluster 2024-03-07T03:13:19.994121+0000 mon.a (mon.0) 858 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-03-07T03:13:21.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:19.995108+0000 mon.a (mon.0) 859 : audit 1 from='client.? 172.21.15.148:0/2618491298' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:21.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:20.392908+0000 mon.a (mon.0) 860 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:20.404805+0000 mon.a (mon.0) 861 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:20.700807+0000 mon.a (mon.0) 862 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:20.716941+0000 mon.a (mon.0) 863 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:21.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:20.718525+0000 mon.b (mon.1) 121 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:21.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:20.720752+0000 mon.b (mon.1) 122 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:21.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:21 smithi195 bash[23221]: audit 2024-03-07T03:13:20.735669+0000 mon.a (mon.0) 864 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:22.287 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[21362]: audit 2024-03-07T03:13:21.001194+0000 mon.a (mon.0) 865 : audit 1 from='client.? 172.21.15.148:0/2618491298' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:22.287 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[21362]: cluster 2024-03-07T03:13:21.001807+0000 mon.a (mon.0) 866 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-03-07T03:13:22.287 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[21362]: cluster 2024-03-07T03:13:21.057470+0000 mgr.y (mgr.24287) 178 : cluster 0 pgmap v158: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:22.287 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[21362]: cluster 2024-03-07T03:13:21.997865+0000 mon.a (mon.0) 867 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-03-07T03:13:22.287 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[31149]: audit 2024-03-07T03:13:21.001194+0000 mon.a (mon.0) 865 : audit 1 from='client.? 172.21.15.148:0/2618491298' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:22.287 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[31149]: cluster 2024-03-07T03:13:21.001807+0000 mon.a (mon.0) 866 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-03-07T03:13:22.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[31149]: cluster 2024-03-07T03:13:21.057470+0000 mgr.y (mgr.24287) 178 : cluster 0 pgmap v158: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:22.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:22 smithi148 bash[31149]: cluster 2024-03-07T03:13:21.997865+0000 mon.a (mon.0) 867 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-03-07T03:13:22.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:22 smithi195 bash[23221]: audit 2024-03-07T03:13:21.001194+0000 mon.a (mon.0) 865 : audit 1 from='client.? 172.21.15.148:0/2618491298' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:22.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:22 smithi195 bash[23221]: cluster 2024-03-07T03:13:21.001807+0000 mon.a (mon.0) 866 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-03-07T03:13:22.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:22 smithi195 bash[23221]: cluster 2024-03-07T03:13:21.057470+0000 mgr.y (mgr.24287) 178 : cluster 0 pgmap v158: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:22.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:22 smithi195 bash[23221]: cluster 2024-03-07T03:13:21.997865+0000 mon.a (mon.0) 867 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-03-07T03:13:23.016 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:109: Skipped 2024-03-07T03:13:23.016 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.016 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.AioDiscardEvent (2024 ms) 2024-03-07T03:13:23.016 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-03-07T03:13:23.039 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:200: Skipped 2024-03-07T03:13:23.040 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.040 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.AioWriteEvent (24 ms) 2024-03-07T03:13:23.040 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-03-07T03:13:23.061 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:261: Skipped 2024-03-07T03:13:23.061 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.062 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.AioFlushEvent (22 ms) 2024-03-07T03:13:23.062 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-03-07T03:13:23.083 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:307: Skipped 2024-03-07T03:13:23.083 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.083 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapCreate (22 ms) 2024-03-07T03:13:23.084 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-03-07T03:13:23.105 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:348: Skipped 2024-03-07T03:13:23.105 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.105 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapProtect (21 ms) 2024-03-07T03:13:23.105 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-03-07T03:13:23.125 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:394: Skipped 2024-03-07T03:13:23.125 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.125 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapUnprotect (20 ms) 2024-03-07T03:13:23.125 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-03-07T03:13:23.144 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:450: Skipped 2024-03-07T03:13:23.144 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.144 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapRename (19 ms) 2024-03-07T03:13:23.144 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-03-07T03:13:23.164 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:500: Skipped 2024-03-07T03:13:23.164 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.164 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapRollback (20 ms) 2024-03-07T03:13:23.164 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-03-07T03:13:23.183 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:541: Skipped 2024-03-07T03:13:23.183 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.183 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapRemove (19 ms) 2024-03-07T03:13:23.183 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.Rename 2024-03-07T03:13:23.202 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:590: Skipped 2024-03-07T03:13:23.202 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.Rename (19 ms) 2024-03-07T03:13:23.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.Resize 2024-03-07T03:13:23.220 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:624: Skipped 2024-03-07T03:13:23.220 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.221 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.Resize (18 ms) 2024-03-07T03:13:23.221 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.Flatten 2024-03-07T03:13:23.240 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:657: Skipped 2024-03-07T03:13:23.240 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.240 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.Flatten (20 ms) 2024-03-07T03:13:23.240 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-03-07T03:13:23.261 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:704: Skipped 2024-03-07T03:13:23.261 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.261 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.UpdateFeatures (21 ms) 2024-03-07T03:13:23.262 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-03-07T03:13:23.281 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:741: Skipped 2024-03-07T03:13:23.282 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.282 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.MetadataSet (20 ms) 2024-03-07T03:13:23.282 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-03-07T03:13:23.302 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:781: Skipped 2024-03-07T03:13:23.302 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.302 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.MetadataRemove (20 ms) 2024-03-07T03:13:23.302 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-03-07T03:13:23.322 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Replay.cc:824: Skipped 2024-03-07T03:13:23.322 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:23.322 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.ObjectPosition (20 ms) 2024-03-07T03:13:23.352 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-07T03:13:24.274 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 16 tests from TestJournalReplay (2330 ms total) 2024-03-07T03:13:24.274 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:13:24.274 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 4 tests from TestJournalEntries 2024-03-07T03:13:24.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:24 smithi148 bash[21362]: cluster 2024-03-07T03:13:23.004825+0000 mon.a (mon.0) 868 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-03-07T03:13:24.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:24 smithi148 bash[31149]: cluster 2024-03-07T03:13:23.004825+0000 mon.a (mon.0) 868 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-03-07T03:13:24.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:24 smithi195 bash[23221]: cluster 2024-03-07T03:13:23.004825+0000 mon.a (mon.0) 868 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-03-07T03:13:24.994 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:24 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:13:24] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2024-03-07T03:13:25.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:25 smithi148 bash[21362]: cluster 2024-03-07T03:13:23.058230+0000 mgr.y (mgr.24287) 179 : cluster 0 pgmap v161: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:25.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:25 smithi148 bash[21362]: cluster 2024-03-07T03:13:24.280052+0000 mon.a (mon.0) 869 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-03-07T03:13:25.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:25 smithi148 bash[31149]: cluster 2024-03-07T03:13:23.058230+0000 mgr.y (mgr.24287) 179 : cluster 0 pgmap v161: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:25.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:25 smithi148 bash[31149]: cluster 2024-03-07T03:13:24.280052+0000 mon.a (mon.0) 869 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-03-07T03:13:25.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:25 smithi195 bash[23221]: cluster 2024-03-07T03:13:23.058230+0000 mgr.y (mgr.24287) 179 : cluster 0 pgmap v161: 105 pgs: 32 creating+peering, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:25.315 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:25 smithi195 bash[23221]: cluster 2024-03-07T03:13:24.280052+0000 mon.a (mon.0) 869 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-03-07T03:13:26.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-03-07T03:13:26.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:26 smithi148 bash[21362]: cluster 2024-03-07T03:13:25.059411+0000 mgr.y (mgr.24287) 180 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:26.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:26 smithi148 bash[21362]: cluster 2024-03-07T03:13:25.284928+0000 mon.a (mon.0) 870 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-03-07T03:13:26.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:26 smithi148 bash[21362]: audit 2024-03-07T03:13:25.292122+0000 mon.a (mon.0) 871 : audit 1 from='client.? 172.21.15.148:0/2060505773' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:26.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:26 smithi148 bash[31149]: cluster 2024-03-07T03:13:25.059411+0000 mgr.y (mgr.24287) 180 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:26.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:26 smithi148 bash[31149]: cluster 2024-03-07T03:13:25.284928+0000 mon.a (mon.0) 870 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-03-07T03:13:26.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:26 smithi148 bash[31149]: audit 2024-03-07T03:13:25.292122+0000 mon.a (mon.0) 871 : audit 1 from='client.? 172.21.15.148:0/2060505773' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:26.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:26 smithi195 bash[23221]: cluster 2024-03-07T03:13:25.059411+0000 mgr.y (mgr.24287) 180 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:26.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:26 smithi195 bash[23221]: cluster 2024-03-07T03:13:25.284928+0000 mon.a (mon.0) 870 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-03-07T03:13:26.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:26 smithi195 bash[23221]: audit 2024-03-07T03:13:25.292122+0000 mon.a (mon.0) 871 : audit 1 from='client.? 172.21.15.148:0/2060505773' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:27.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:27 smithi148 bash[21362]: audit 2024-03-07T03:13:26.296212+0000 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.148:0/2060505773' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:27.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:27 smithi148 bash[21362]: cluster 2024-03-07T03:13:26.296618+0000 mon.a (mon.0) 873 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-03-07T03:13:27.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:27 smithi148 bash[31149]: audit 2024-03-07T03:13:26.296212+0000 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.148:0/2060505773' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:27.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:27 smithi148 bash[31149]: cluster 2024-03-07T03:13:26.296618+0000 mon.a (mon.0) 873 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-03-07T03:13:27.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:27 smithi195 bash[23221]: audit 2024-03-07T03:13:26.296212+0000 mon.a (mon.0) 872 : audit 1 from='client.? 172.21.15.148:0/2060505773' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:27.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:27 smithi195 bash[23221]: cluster 2024-03-07T03:13:26.296618+0000 mon.a (mon.0) 873 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-03-07T03:13:28.084 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:13:28.338 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Entries.cc:118: Skipped 2024-03-07T03:13:28.339 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:28.339 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioWrite (2050 ms) 2024-03-07T03:13:28.339 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-03-07T03:13:28.359 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Entries.cc:163: Skipped 2024-03-07T03:13:28.359 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:28.360 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioDiscard (21 ms) 2024-03-07T03:13:28.360 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalEntries.AioDiscardWithPrune 2024-03-07T03:13:28.381 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Entries.cc:200: Skipped 2024-03-07T03:13:28.381 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:28.381 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioDiscardWithPrune (22 ms) 2024-03-07T03:13:28.382 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-03-07T03:13:28.400 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/journal/test_Entries.cc:263: Skipped 2024-03-07T03:13:28.401 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:28.401 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioFlush (19 ms) 2024-03-07T03:13:28.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:28 smithi148 bash[21362]: cluster 2024-03-07T03:13:27.060132+0000 mgr.y (mgr.24287) 181 : cluster 0 pgmap v166: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:28.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:28 smithi148 bash[21362]: cluster 2024-03-07T03:13:27.307798+0000 mon.a (mon.0) 874 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-03-07T03:13:28.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:28 smithi148 bash[31149]: cluster 2024-03-07T03:13:27.060132+0000 mgr.y (mgr.24287) 181 : cluster 0 pgmap v166: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:28.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:28 smithi148 bash[31149]: cluster 2024-03-07T03:13:27.307798+0000 mon.a (mon.0) 874 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-03-07T03:13:28.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:28 smithi195 bash[23221]: cluster 2024-03-07T03:13:27.060132+0000 mgr.y (mgr.24287) 181 : cluster 0 pgmap v166: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:28.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:28 smithi195 bash[23221]: cluster 2024-03-07T03:13:27.307798+0000 mon.a (mon.0) 874 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-03-07T03:13:29.325 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 4 tests from TestJournalEntries (2112 ms total) 2024-03-07T03:13:29.326 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:13:29.326 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 2 tests from TestTrash 2024-03-07T03:13:29.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:29 smithi195 bash[23221]: cluster 2024-03-07T03:13:28.319961+0000 mon.a (mon.0) 875 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-03-07T03:13:29.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:29 smithi148 bash[21362]: cluster 2024-03-07T03:13:28.319961+0000 mon.a (mon.0) 875 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-03-07T03:13:29.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:29 smithi148 bash[31149]: cluster 2024-03-07T03:13:28.319961+0000 mon.a (mon.0) 875 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-03-07T03:13:30.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:30 smithi195 bash[23221]: cluster 2024-03-07T03:13:29.060666+0000 mgr.y (mgr.24287) 182 : cluster 0 pgmap v169: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:30.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:30 smithi195 bash[23221]: cluster 2024-03-07T03:13:29.333995+0000 mon.a (mon.0) 876 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-03-07T03:13:30.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:30 smithi148 bash[21362]: cluster 2024-03-07T03:13:29.060666+0000 mgr.y (mgr.24287) 182 : cluster 0 pgmap v169: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:30.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:30 smithi148 bash[21362]: cluster 2024-03-07T03:13:29.333995+0000 mon.a (mon.0) 876 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-03-07T03:13:30.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:30 smithi148 bash[31149]: cluster 2024-03-07T03:13:29.060666+0000 mgr.y (mgr.24287) 182 : cluster 0 pgmap v169: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:30.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:30 smithi148 bash[31149]: cluster 2024-03-07T03:13:29.333995+0000 mon.a (mon.0) 876 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-03-07T03:13:30.859 INFO:teuthology.orchestra.run.smithi148.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:13:31.338 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-03-07T03:13:31.516 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:13:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[21362]: cluster 2024-03-07T03:13:30.342430+0000 mon.a (mon.0) 877 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-03-07T03:13:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[21362]: audit 2024-03-07T03:13:30.356882+0000 mon.c (mon.2) 32 : audit 1 from='client.? 172.21.15.148:0/1955317974' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[21362]: audit 2024-03-07T03:13:30.357852+0000 mon.a (mon.0) 878 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[21362]: audit 2024-03-07T03:13:30.863732+0000 mgr.y (mgr.24287) 183 : audit 0 from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[21362]: cephadm 2024-03-07T03:13:30.864559+0000 mgr.y (mgr.24287) 184 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[21362]: audit 2024-03-07T03:13:30.871799+0000 mon.a (mon.0) 879 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[21362]: audit 2024-03-07T03:13:30.903278+0000 mon.b (mon.1) 123 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[31149]: cluster 2024-03-07T03:13:30.342430+0000 mon.a (mon.0) 877 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[31149]: audit 2024-03-07T03:13:30.356882+0000 mon.c (mon.2) 32 : audit 1 from='client.? 172.21.15.148:0/1955317974' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[31149]: audit 2024-03-07T03:13:30.357852+0000 mon.a (mon.0) 878 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[31149]: audit 2024-03-07T03:13:30.863732+0000 mgr.y (mgr.24287) 183 : audit 0 from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[31149]: cephadm 2024-03-07T03:13:30.864559+0000 mgr.y (mgr.24287) 184 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[31149]: audit 2024-03-07T03:13:30.871799+0000 mon.a (mon.0) 879 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:31 smithi148 bash[31149]: audit 2024-03-07T03:13:30.903278+0000 mon.b (mon.1) 123 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:31.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:31 smithi195 bash[23221]: cluster 2024-03-07T03:13:30.342430+0000 mon.a (mon.0) 877 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-03-07T03:13:31.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:31 smithi195 bash[23221]: audit 2024-03-07T03:13:30.356882+0000 mon.c (mon.2) 32 : audit 1 from='client.? 172.21.15.148:0/1955317974' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:31.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:31 smithi195 bash[23221]: audit 2024-03-07T03:13:30.357852+0000 mon.a (mon.0) 878 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:31.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:31 smithi195 bash[23221]: audit 2024-03-07T03:13:30.863732+0000 mgr.y (mgr.24287) 183 : audit 0 from='client.14796 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:13:31.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:31 smithi195 bash[23221]: cephadm 2024-03-07T03:13:30.864559+0000 mgr.y (mgr.24287) 184 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:13:31.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:31 smithi195 bash[23221]: audit 2024-03-07T03:13:30.871799+0000 mon.a (mon.0) 879 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:31.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:31 smithi195 bash[23221]: audit 2024-03-07T03:13:30.903278+0000 mon.b (mon.1) 123 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: cluster 2024-03-07T03:13:31.061289+0000 mgr.y (mgr.24287) 185 : cluster 0 pgmap v172: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: audit 2024-03-07T03:13:31.349258+0000 mon.a (mon.0) 880 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: cluster 2024-03-07T03:13:31.349399+0000 mon.a (mon.0) 881 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: audit 2024-03-07T03:13:31.426672+0000 mon.b (mon.1) 124 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: audit 2024-03-07T03:13:31.428488+0000 mon.b (mon.1) 125 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: audit 2024-03-07T03:13:31.443268+0000 mon.a (mon.0) 882 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: cephadm 2024-03-07T03:13:31.506744+0000 mgr.y (mgr.24287) 186 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:13:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[31149]: cluster 2024-03-07T03:13:31.867893+0000 mon.a (mon.0) 883 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: cluster 2024-03-07T03:13:31.061289+0000 mgr.y (mgr.24287) 185 : cluster 0 pgmap v172: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: audit 2024-03-07T03:13:31.349258+0000 mon.a (mon.0) 880 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: cluster 2024-03-07T03:13:31.349399+0000 mon.a (mon.0) 881 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: audit 2024-03-07T03:13:31.426672+0000 mon.b (mon.1) 124 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: audit 2024-03-07T03:13:31.428488+0000 mon.b (mon.1) 125 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: audit 2024-03-07T03:13:31.443268+0000 mon.a (mon.0) 882 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: cephadm 2024-03-07T03:13:31.506744+0000 mgr.y (mgr.24287) 186 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:13:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:32 smithi148 bash[21362]: cluster 2024-03-07T03:13:31.867893+0000 mon.a (mon.0) 883 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: cluster 2024-03-07T03:13:31.061289+0000 mgr.y (mgr.24287) 185 : cluster 0 pgmap v172: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: audit 2024-03-07T03:13:31.349258+0000 mon.a (mon.0) 880 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: cluster 2024-03-07T03:13:31.349399+0000 mon.a (mon.0) 881 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: audit 2024-03-07T03:13:31.426672+0000 mon.b (mon.1) 124 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: audit 2024-03-07T03:13:31.428488+0000 mon.b (mon.1) 125 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: audit 2024-03-07T03:13:31.443268+0000 mon.a (mon.0) 882 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: cephadm 2024-03-07T03:13:31.506744+0000 mgr.y (mgr.24287) 186 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:ea9a5688d9980e4ddedb59679d2960bcbac6fc71 2024-03-07T03:13:32.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:32 smithi195 bash[23221]: cluster 2024-03-07T03:13:31.867893+0000 mon.a (mon.0) 883 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:33.509 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestTrash.UserRemovingSource (2170 ms) 2024-03-07T03:13:33.509 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-03-07T03:13:33.592 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestTrash.RestoreMirroringSource (83 ms) 2024-03-07T03:13:33.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:33 smithi148 bash[21362]: cluster 2024-03-07T03:13:32.363937+0000 mon.a (mon.0) 884 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-03-07T03:13:33.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:33 smithi148 bash[31149]: cluster 2024-03-07T03:13:32.363937+0000 mon.a (mon.0) 884 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-03-07T03:13:33.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:33 smithi195 bash[23221]: cluster 2024-03-07T03:13:32.363937+0000 mon.a (mon.0) 884 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-03-07T03:13:34.389 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 2 tests from TestTrash (2253 ms total) 2024-03-07T03:13:34.389 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:13:34.389 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 1 test from TestOperations 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[31149]: cluster 2024-03-07T03:13:33.062133+0000 mgr.y (mgr.24287) 187 : cluster 0 pgmap v175: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[31149]: cluster 2024-03-07T03:13:33.358407+0000 mon.a (mon.0) 885 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[31149]: cluster 2024-03-07T03:13:33.358451+0000 mon.a (mon.0) 886 : cluster 1 Cluster is now healthy 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[31149]: cluster 2024-03-07T03:13:33.364246+0000 mon.a (mon.0) 887 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[31149]: audit 2024-03-07T03:13:34.049469+0000 mon.b (mon.1) 126 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[21362]: cluster 2024-03-07T03:13:33.062133+0000 mgr.y (mgr.24287) 187 : cluster 0 pgmap v175: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[21362]: cluster 2024-03-07T03:13:33.358407+0000 mon.a (mon.0) 885 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[21362]: cluster 2024-03-07T03:13:33.358451+0000 mon.a (mon.0) 886 : cluster 1 Cluster is now healthy 2024-03-07T03:13:34.632 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[21362]: cluster 2024-03-07T03:13:33.364246+0000 mon.a (mon.0) 887 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-03-07T03:13:34.633 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[21362]: audit 2024-03-07T03:13:34.049469+0000 mon.b (mon.1) 126 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:34.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:34 smithi195 bash[23221]: cluster 2024-03-07T03:13:33.062133+0000 mgr.y (mgr.24287) 187 : cluster 0 pgmap v175: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 220 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:34.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:34 smithi195 bash[23221]: cluster 2024-03-07T03:13:33.358407+0000 mon.a (mon.0) 885 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:34.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:34 smithi195 bash[23221]: cluster 2024-03-07T03:13:33.358451+0000 mon.a (mon.0) 886 : cluster 1 Cluster is now healthy 2024-03-07T03:13:34.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:34 smithi195 bash[23221]: cluster 2024-03-07T03:13:33.364246+0000 mon.a (mon.0) 887 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-03-07T03:13:34.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:34 smithi195 bash[23221]: audit 2024-03-07T03:13:34.049469+0000 mon.b (mon.1) 126 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:35.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:34 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:13:34] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.43.0" 2024-03-07T03:13:35.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:35 smithi148 bash[21362]: cluster 2024-03-07T03:13:34.390905+0000 mon.a (mon.0) 888 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-03-07T03:13:35.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:35 smithi148 bash[31149]: cluster 2024-03-07T03:13:34.390905+0000 mon.a (mon.0) 888 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-03-07T03:13:35.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:35 smithi195 bash[23221]: cluster 2024-03-07T03:13:34.390905+0000 mon.a (mon.0) 888 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-03-07T03:13:36.254 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:13:36.389 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-03-07T03:13:36.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[31149]: cluster 2024-03-07T03:13:35.063302+0000 mgr.y (mgr.24287) 188 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:36.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[31149]: cluster 2024-03-07T03:13:35.411728+0000 mon.a (mon.0) 889 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2024-03-07T03:13:36.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[31149]: audit 2024-03-07T03:13:35.414343+0000 mon.c (mon.2) 33 : audit 1 from='client.? 172.21.15.148:0/2728201445' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:36.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[31149]: audit 2024-03-07T03:13:35.415767+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:36.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[31149]: audit 2024-03-07T03:13:36.401042+0000 mon.a (mon.0) 891 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:36.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[31149]: cluster 2024-03-07T03:13:36.401244+0000 mon.a (mon.0) 892 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2024-03-07T03:13:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[21362]: cluster 2024-03-07T03:13:35.063302+0000 mgr.y (mgr.24287) 188 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[21362]: cluster 2024-03-07T03:13:35.411728+0000 mon.a (mon.0) 889 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2024-03-07T03:13:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[21362]: audit 2024-03-07T03:13:35.414343+0000 mon.c (mon.2) 33 : audit 1 from='client.? 172.21.15.148:0/2728201445' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[21362]: audit 2024-03-07T03:13:35.415767+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[21362]: audit 2024-03-07T03:13:36.401042+0000 mon.a (mon.0) 891 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:36 smithi148 bash[21362]: cluster 2024-03-07T03:13:36.401244+0000 mon.a (mon.0) 892 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2024-03-07T03:13:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:36 smithi195 bash[23221]: cluster 2024-03-07T03:13:35.063302+0000 mgr.y (mgr.24287) 188 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:36 smithi195 bash[23221]: cluster 2024-03-07T03:13:35.411728+0000 mon.a (mon.0) 889 : cluster 0 osdmap e93: 8 total, 8 up, 8 in 2024-03-07T03:13:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:36 smithi195 bash[23221]: audit 2024-03-07T03:13:35.414343+0000 mon.c (mon.2) 33 : audit 1 from='client.? 172.21.15.148:0/2728201445' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:36 smithi195 bash[23221]: audit 2024-03-07T03:13:35.415767+0000 mon.a (mon.0) 890 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:36 smithi195 bash[23221]: audit 2024-03-07T03:13:36.401042+0000 mon.a (mon.0) 891 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:36 smithi195 bash[23221]: cluster 2024-03-07T03:13:36.401244+0000 mon.a (mon.0) 892 : cluster 0 osdmap e94: 8 total, 8 up, 8 in 2024-03-07T03:13:38.433 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_Operations.cc:18: Skipped 2024-03-07T03:13:38.433 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:13:38.433 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestOperations.DisableJournalingCorrupt (2044 ms) 2024-03-07T03:13:38.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:38 smithi148 bash[21362]: cluster 2024-03-07T03:13:37.063982+0000 mgr.y (mgr.24287) 189 : cluster 0 pgmap v181: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:38.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:38 smithi148 bash[21362]: cluster 2024-03-07T03:13:37.401018+0000 mon.a (mon.0) 893 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2024-03-07T03:13:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:38 smithi148 bash[31149]: cluster 2024-03-07T03:13:37.063982+0000 mgr.y (mgr.24287) 189 : cluster 0 pgmap v181: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:38 smithi148 bash[31149]: cluster 2024-03-07T03:13:37.401018+0000 mon.a (mon.0) 893 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2024-03-07T03:13:38.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:38 smithi195 bash[23221]: cluster 2024-03-07T03:13:37.063982+0000 mgr.y (mgr.24287) 189 : cluster 0 pgmap v181: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:38.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:38 smithi195 bash[23221]: cluster 2024-03-07T03:13:37.401018+0000 mon.a (mon.0) 893 : cluster 0 osdmap e95: 8 total, 8 up, 8 in 2024-03-07T03:13:39.413 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 1 test from TestOperations (2044 ms total) 2024-03-07T03:13:39.413 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:13:39.413 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 4 tests from TestObjectMap 2024-03-07T03:13:39.730 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:39 smithi148 bash[21362]: cluster 2024-03-07T03:13:38.412838+0000 mon.a (mon.0) 894 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2024-03-07T03:13:39.730 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:39 smithi148 bash[31149]: cluster 2024-03-07T03:13:38.412838+0000 mon.a (mon.0) 894 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2024-03-07T03:13:39.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:39 smithi195 bash[23221]: cluster 2024-03-07T03:13:38.412838+0000 mon.a (mon.0) 894 : cluster 0 osdmap e96: 8 total, 8 up, 8 in 2024-03-07T03:13:40.208 INFO:teuthology.orchestra.run.smithi148.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:alertmanager.a smithi148 *:9093,9094 running (3m) 19s ago 4m 15.1M - 0.25.0 c8568f914cd2 a6ee726aca40 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:grafana.a smithi195 *:3000 running (3m) 19s ago 3m 79.5M - 9.4.7 954c08fa6188 a09a8cb7fec4 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi148.qizikc smithi148 running (2m) 19s ago 2m 11.9M - 18.2.1-718-gae75fc9d 0164bb8a7c67 d82cf82f7898 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi195.gtlmjn smithi195 running (2m) 19s ago 2m 13.3M - 18.2.1-718-gae75fc9d 0164bb8a7c67 f9ba5c67a194 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.x smithi195 *:8443,8765 running (12m) 19s ago 12m 436M - 18.2.1-718-gae75fc9d 0164bb8a7c67 1e49947f43d7 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.y smithi148 *:9283,8765 running (14m) 19s ago 14m 490M - 18.2.1-718-gae75fc9d 0164bb8a7c67 d7042aa9c7c7 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:mon.a smithi148 running (14m) 19s ago 14m 48.7M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 68e42a3c8dcc 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:mon.b smithi195 running (12m) 19s ago 12m 40.5M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 aa6d6fcf0602 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:mon.c smithi148 running (12m) 19s ago 12m 38.4M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 683a6360bfe4 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.a smithi148 *:9100 running (4m) 19s ago 4m 8059k - 1.5.0 0da6a335fe13 d9b895f99d5c 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.b smithi195 *:9100 running (4m) 19s ago 4m 8179k - 1.5.0 0da6a335fe13 be49ab11d18e 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:osd.0 smithi148 running (11m) 19s ago 11m 46.4M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 f8c58575ec57 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:osd.1 smithi148 running (10m) 19s ago 10m 45.8M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 3125450ae62f 2024-03-07T03:13:40.209 INFO:teuthology.orchestra.run.smithi148.stdout:osd.2 smithi148 running (10m) 19s ago 10m 43.7M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 b463a715bf36 2024-03-07T03:13:40.210 INFO:teuthology.orchestra.run.smithi148.stdout:osd.3 smithi148 running (9m) 19s ago 9m 42.6M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 180f02eefffa 2024-03-07T03:13:40.210 INFO:teuthology.orchestra.run.smithi148.stdout:osd.4 smithi195 running (8m) 19s ago 8m 44.1M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 5023ce93366a 2024-03-07T03:13:40.210 INFO:teuthology.orchestra.run.smithi148.stdout:osd.5 smithi195 running (7m) 19s ago 7m 42.8M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 c850d7f8ae40 2024-03-07T03:13:40.210 INFO:teuthology.orchestra.run.smithi148.stdout:osd.6 smithi195 running (6m) 19s ago 6m 41.2M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 332f73f59fdc 2024-03-07T03:13:40.210 INFO:teuthology.orchestra.run.smithi148.stdout:osd.7 smithi195 running (5m) 19s ago 5m 43.3M 4060M 18.2.1-718-gae75fc9d 0164bb8a7c67 9a7e2329950b 2024-03-07T03:13:40.210 INFO:teuthology.orchestra.run.smithi148.stdout:prometheus.a smithi195 *:9095 running (3m) 19s ago 5m 40.8M - 2.43.0 a07b618ecd1d 7949aad1132f 2024-03-07T03:13:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:40 smithi148 bash[21362]: cluster 2024-03-07T03:13:39.064768+0000 mgr.y (mgr.24287) 190 : cluster 0 pgmap v184: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:40 smithi148 bash[21362]: cluster 2024-03-07T03:13:39.420518+0000 mon.a (mon.0) 895 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2024-03-07T03:13:40.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:40 smithi148 bash[31149]: cluster 2024-03-07T03:13:39.064768+0000 mgr.y (mgr.24287) 190 : cluster 0 pgmap v184: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:40.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:40 smithi148 bash[31149]: cluster 2024-03-07T03:13:39.420518+0000 mon.a (mon.0) 895 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2024-03-07T03:13:40.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:40 smithi195 bash[23221]: cluster 2024-03-07T03:13:39.064768+0000 mgr.y (mgr.24287) 190 : cluster 0 pgmap v184: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:40.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:40 smithi195 bash[23221]: cluster 2024-03-07T03:13:39.420518+0000 mon.a (mon.0) 895 : cluster 0 osdmap e97: 8 total, 8 up, 8 in 2024-03-07T03:13:41.089 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for minority of mons to upgrade' 2024-03-07T03:13:41.453 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-03-07T03:13:41.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[21362]: audit 2024-03-07T03:13:40.182891+0000 mgr.y (mgr.24287) 191 : audit 0 from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:13:41.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[21362]: cluster 2024-03-07T03:13:40.428348+0000 mon.a (mon.0) 896 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2024-03-07T03:13:41.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[21362]: audit 2024-03-07T03:13:40.439441+0000 mon.c (mon.2) 34 : audit 1 from='client.? 172.21.15.148:0/1203926747' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:41.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[21362]: audit 2024-03-07T03:13:40.439839+0000 mon.a (mon.0) 897 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:41.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[31149]: audit 2024-03-07T03:13:40.182891+0000 mgr.y (mgr.24287) 191 : audit 0 from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:13:41.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[31149]: cluster 2024-03-07T03:13:40.428348+0000 mon.a (mon.0) 896 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2024-03-07T03:13:41.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[31149]: audit 2024-03-07T03:13:40.439441+0000 mon.c (mon.2) 34 : audit 1 from='client.? 172.21.15.148:0/1203926747' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:41.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:41 smithi148 bash[31149]: audit 2024-03-07T03:13:40.439839+0000 mon.a (mon.0) 897 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:41.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:41 smithi195 bash[23221]: audit 2024-03-07T03:13:40.182891+0000 mgr.y (mgr.24287) 191 : audit 0 from='client.24734 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:13:41.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:41 smithi195 bash[23221]: cluster 2024-03-07T03:13:40.428348+0000 mon.a (mon.0) 896 : cluster 0 osdmap e98: 8 total, 8 up, 8 in 2024-03-07T03:13:41.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:41 smithi195 bash[23221]: audit 2024-03-07T03:13:40.439441+0000 mon.c (mon.2) 34 : audit 1 from='client.? 172.21.15.148:0/1203926747' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:41.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:41 smithi195 bash[23221]: audit 2024-03-07T03:13:40.439839+0000 mon.a (mon.0) 897 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:42.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[21362]: cluster 2024-03-07T03:13:41.065733+0000 mgr.y (mgr.24287) 192 : cluster 0 pgmap v187: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:42.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[21362]: cluster 2024-03-07T03:13:41.425117+0000 mon.a (mon.0) 898 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[21362]: audit 2024-03-07T03:13:41.434669+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[21362]: cluster 2024-03-07T03:13:41.435001+0000 mon.a (mon.0) 900 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2024-03-07T03:13:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[31149]: cluster 2024-03-07T03:13:41.065733+0000 mgr.y (mgr.24287) 192 : cluster 0 pgmap v187: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[31149]: cluster 2024-03-07T03:13:41.425117+0000 mon.a (mon.0) 898 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[31149]: audit 2024-03-07T03:13:41.434669+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:42 smithi148 bash[31149]: cluster 2024-03-07T03:13:41.435001+0000 mon.a (mon.0) 900 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2024-03-07T03:13:42.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:42 smithi195 bash[23221]: cluster 2024-03-07T03:13:41.065733+0000 mgr.y (mgr.24287) 192 : cluster 0 pgmap v187: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:42.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:42 smithi195 bash[23221]: cluster 2024-03-07T03:13:41.425117+0000 mon.a (mon.0) 898 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:42.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:42 smithi195 bash[23221]: audit 2024-03-07T03:13:41.434669+0000 mon.a (mon.0) 899 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:42.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:42 smithi195 bash[23221]: cluster 2024-03-07T03:13:41.435001+0000 mon.a (mon.0) 900 : cluster 0 osdmap e99: 8 total, 8 up, 8 in 2024-03-07T03:13:43.537 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2084 ms) 2024-03-07T03:13:43.537 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-03-07T03:13:43.584 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (48 ms) 2024-03-07T03:13:43.584 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-03-07T03:13:43.661 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (76 ms) 2024-03-07T03:13:43.661 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-03-07T03:13:43.715 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (54 ms) 2024-03-07T03:13:43.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:43 smithi148 bash[21362]: cluster 2024-03-07T03:13:42.454716+0000 mon.a (mon.0) 901 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2024-03-07T03:13:43.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:43 smithi148 bash[31149]: cluster 2024-03-07T03:13:42.454716+0000 mon.a (mon.0) 901 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2024-03-07T03:13:43.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:43 smithi195 bash[23221]: cluster 2024-03-07T03:13:42.454716+0000 mon.a (mon.0) 901 : cluster 0 osdmap e100: 8 total, 8 up, 8 in 2024-03-07T03:13:44.478 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 4 tests from TestObjectMap (2262 ms total) 2024-03-07T03:13:44.478 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:13:44.478 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 2 tests from TestMirroringWatcher 2024-03-07T03:13:44.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:13:44] "GET /metrics HTTP/1.1" 200 36216 "" "Prometheus/2.43.0" 2024-03-07T03:13:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[31149]: cluster 2024-03-07T03:13:43.066436+0000 mgr.y (mgr.24287) 193 : cluster 0 pgmap v190: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[31149]: cluster 2024-03-07T03:13:43.461815+0000 mon.a (mon.0) 902 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[31149]: cluster 2024-03-07T03:13:43.461864+0000 mon.a (mon.0) 903 : cluster 1 Cluster is now healthy 2024-03-07T03:13:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[31149]: cluster 2024-03-07T03:13:43.484758+0000 mon.a (mon.0) 904 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2024-03-07T03:13:44.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[21362]: cluster 2024-03-07T03:13:43.066436+0000 mgr.y (mgr.24287) 193 : cluster 0 pgmap v190: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:44.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[21362]: cluster 2024-03-07T03:13:43.461815+0000 mon.a (mon.0) 902 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:44.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[21362]: cluster 2024-03-07T03:13:43.461864+0000 mon.a (mon.0) 903 : cluster 1 Cluster is now healthy 2024-03-07T03:13:44.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:44 smithi148 bash[21362]: cluster 2024-03-07T03:13:43.484758+0000 mon.a (mon.0) 904 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2024-03-07T03:13:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:44 smithi195 bash[23221]: cluster 2024-03-07T03:13:43.066436+0000 mgr.y (mgr.24287) 193 : cluster 0 pgmap v190: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 221 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:44 smithi195 bash[23221]: cluster 2024-03-07T03:13:43.461815+0000 mon.a (mon.0) 902 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:13:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:44 smithi195 bash[23221]: cluster 2024-03-07T03:13:43.461864+0000 mon.a (mon.0) 903 : cluster 1 Cluster is now healthy 2024-03-07T03:13:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:44 smithi195 bash[23221]: cluster 2024-03-07T03:13:43.484758+0000 mon.a (mon.0) 904 : cluster 0 osdmap e101: 8 total, 8 up, 8 in 2024-03-07T03:13:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:45 smithi148 bash[21362]: cluster 2024-03-07T03:13:44.486810+0000 mon.a (mon.0) 905 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2024-03-07T03:13:45.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:45 smithi148 bash[31149]: cluster 2024-03-07T03:13:44.486810+0000 mon.a (mon.0) 905 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2024-03-07T03:13:45.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:45 smithi195 bash[23221]: cluster 2024-03-07T03:13:44.486810+0000 mon.a (mon.0) 905 : cluster 0 osdmap e102: 8 total, 8 up, 8 in 2024-03-07T03:13:45.833 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:13:46.509 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-03-07T03:13:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:46 smithi148 bash[31149]: cluster 2024-03-07T03:13:45.067214+0000 mgr.y (mgr.24287) 194 : cluster 0 pgmap v193: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:46 smithi148 bash[31149]: cluster 2024-03-07T03:13:45.506388+0000 mon.a (mon.0) 906 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2024-03-07T03:13:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:46 smithi148 bash[31149]: audit 2024-03-07T03:13:45.509583+0000 mon.a (mon.0) 907 : audit 1 from='client.? 172.21.15.148:0/2245558061' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:46 smithi148 bash[21362]: cluster 2024-03-07T03:13:45.067214+0000 mgr.y (mgr.24287) 194 : cluster 0 pgmap v193: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:46 smithi148 bash[21362]: cluster 2024-03-07T03:13:45.506388+0000 mon.a (mon.0) 906 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2024-03-07T03:13:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:46 smithi148 bash[21362]: audit 2024-03-07T03:13:45.509583+0000 mon.a (mon.0) 907 : audit 1 from='client.? 172.21.15.148:0/2245558061' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:46 smithi195 bash[23221]: cluster 2024-03-07T03:13:45.067214+0000 mgr.y (mgr.24287) 194 : cluster 0 pgmap v193: 73 pgs: 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:46 smithi195 bash[23221]: cluster 2024-03-07T03:13:45.506388+0000 mon.a (mon.0) 906 : cluster 0 osdmap e103: 8 total, 8 up, 8 in 2024-03-07T03:13:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:46 smithi195 bash[23221]: audit 2024-03-07T03:13:45.509583+0000 mon.a (mon.0) 907 : audit 1 from='client.? 172.21.15.148:0/2245558061' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:47.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:47 smithi148 bash[21362]: audit 2024-03-07T03:13:46.504841+0000 mon.a (mon.0) 908 : audit 1 from='client.? 172.21.15.148:0/2245558061' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:47.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:47 smithi148 bash[21362]: cluster 2024-03-07T03:13:46.505497+0000 mon.a (mon.0) 909 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2024-03-07T03:13:47.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:47 smithi148 bash[31149]: audit 2024-03-07T03:13:46.504841+0000 mon.a (mon.0) 908 : audit 1 from='client.? 172.21.15.148:0/2245558061' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:47.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:47 smithi148 bash[31149]: cluster 2024-03-07T03:13:46.505497+0000 mon.a (mon.0) 909 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2024-03-07T03:13:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:47 smithi195 bash[23221]: audit 2024-03-07T03:13:46.504841+0000 mon.a (mon.0) 908 : audit 1 from='client.? 172.21.15.148:0/2245558061' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:47.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:47 smithi195 bash[23221]: cluster 2024-03-07T03:13:46.505497+0000 mon.a (mon.0) 909 : cluster 0 osdmap e104: 8 total, 8 up, 8 in 2024-03-07T03:13:48.592 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2083 ms) 2024-03-07T03:13:48.592 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-03-07T03:13:48.627 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (35 ms) 2024-03-07T03:13:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:48 smithi148 bash[21362]: cluster 2024-03-07T03:13:47.068077+0000 mgr.y (mgr.24287) 195 : cluster 0 pgmap v196: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:48 smithi148 bash[21362]: cluster 2024-03-07T03:13:47.536021+0000 mon.a (mon.0) 910 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2024-03-07T03:13:48.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:48 smithi148 bash[31149]: cluster 2024-03-07T03:13:47.068077+0000 mgr.y (mgr.24287) 195 : cluster 0 pgmap v196: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:48.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:48 smithi148 bash[31149]: cluster 2024-03-07T03:13:47.536021+0000 mon.a (mon.0) 910 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2024-03-07T03:13:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:48 smithi195 bash[23221]: cluster 2024-03-07T03:13:47.068077+0000 mgr.y (mgr.24287) 195 : cluster 0 pgmap v196: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:48 smithi195 bash[23221]: cluster 2024-03-07T03:13:47.536021+0000 mon.a (mon.0) 910 : cluster 0 osdmap e105: 8 total, 8 up, 8 in 2024-03-07T03:13:49.545 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 2 tests from TestMirroringWatcher (2118 ms total) 2024-03-07T03:13:49.545 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:13:49.545 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 57 tests from TestMigration 2024-03-07T03:13:49.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:49 smithi148 bash[31149]: cluster 2024-03-07T03:13:48.547146+0000 mon.a (mon.0) 911 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2024-03-07T03:13:49.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:49 smithi148 bash[31149]: audit 2024-03-07T03:13:49.049676+0000 mon.b (mon.1) 127 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:49.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:49 smithi148 bash[21362]: cluster 2024-03-07T03:13:48.547146+0000 mon.a (mon.0) 911 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2024-03-07T03:13:49.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:49 smithi148 bash[21362]: audit 2024-03-07T03:13:49.049676+0000 mon.b (mon.1) 127 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:49 smithi195 bash[23221]: cluster 2024-03-07T03:13:48.547146+0000 mon.a (mon.0) 911 : cluster 0 osdmap e106: 8 total, 8 up, 8 in 2024-03-07T03:13:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:49 smithi195 bash[23221]: audit 2024-03-07T03:13:49.049676+0000 mon.b (mon.1) 127 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:13:49.893 INFO:teuthology.orchestra.run.smithi148.stdout:wait for minority of mons to upgrade 2024-03-07T03:13:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:50 smithi148 bash[21362]: cluster 2024-03-07T03:13:49.069032+0000 mgr.y (mgr.24287) 196 : cluster 0 pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:50 smithi148 bash[21362]: cluster 2024-03-07T03:13:49.550138+0000 mon.a (mon.0) 912 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2024-03-07T03:13:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:50 smithi148 bash[31149]: cluster 2024-03-07T03:13:49.069032+0000 mgr.y (mgr.24287) 196 : cluster 0 pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:50 smithi148 bash[31149]: cluster 2024-03-07T03:13:49.550138+0000 mon.a (mon.0) 912 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2024-03-07T03:13:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:50 smithi195 bash[23221]: cluster 2024-03-07T03:13:49.069032+0000 mgr.y (mgr.24287) 196 : cluster 0 pgmap v199: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 226 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:50 smithi195 bash[23221]: cluster 2024-03-07T03:13:49.550138+0000 mon.a (mon.0) 912 : cluster 0 osdmap e107: 8 total, 8 up, 8 in 2024-03-07T03:13:51.163 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-03-07T03:13:51.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:51 smithi195 bash[23221]: cluster 2024-03-07T03:13:50.563573+0000 mon.a (mon.0) 913 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2024-03-07T03:13:51.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:51 smithi195 bash[23221]: audit 2024-03-07T03:13:50.565044+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.148:0/2568545883' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:52.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:51 smithi148 bash[21362]: cluster 2024-03-07T03:13:50.563573+0000 mon.a (mon.0) 913 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2024-03-07T03:13:52.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:51 smithi148 bash[21362]: audit 2024-03-07T03:13:50.565044+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.148:0/2568545883' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:52.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:51 smithi148 bash[31149]: cluster 2024-03-07T03:13:50.563573+0000 mon.a (mon.0) 913 : cluster 0 osdmap e108: 8 total, 8 up, 8 in 2024-03-07T03:13:52.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:51 smithi148 bash[31149]: audit 2024-03-07T03:13:50.565044+0000 mon.a (mon.0) 914 : audit 1 from='client.? 172.21.15.148:0/2568545883' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:13:52.570 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.Empty 2024-03-07T03:13:52.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:52 smithi195 bash[23221]: cluster 2024-03-07T03:13:51.069514+0000 mgr.y (mgr.24287) 197 : cluster 0 pgmap v202: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:52.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:52 smithi195 bash[23221]: cluster 2024-03-07T03:13:51.558064+0000 mon.a (mon.0) 915 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:52.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:52 smithi195 bash[23221]: audit 2024-03-07T03:13:51.573948+0000 mon.a (mon.0) 916 : audit 1 from='client.? 172.21.15.148:0/2568545883' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:52.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:52 smithi195 bash[23221]: cluster 2024-03-07T03:13:51.574122+0000 mon.a (mon.0) 917 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[31149]: cluster 2024-03-07T03:13:51.069514+0000 mgr.y (mgr.24287) 197 : cluster 0 pgmap v202: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[31149]: cluster 2024-03-07T03:13:51.558064+0000 mon.a (mon.0) 915 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[31149]: audit 2024-03-07T03:13:51.573948+0000 mon.a (mon.0) 916 : audit 1 from='client.? 172.21.15.148:0/2568545883' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[31149]: cluster 2024-03-07T03:13:51.574122+0000 mon.a (mon.0) 917 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[21362]: cluster 2024-03-07T03:13:51.069514+0000 mgr.y (mgr.24287) 197 : cluster 0 pgmap v202: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[21362]: cluster 2024-03-07T03:13:51.558064+0000 mon.a (mon.0) 915 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[21362]: audit 2024-03-07T03:13:51.573948+0000 mon.a (mon.0) 916 : audit 1 from='client.? 172.21.15.148:0/2568545883' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:13:53.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:52 smithi148 bash[21362]: cluster 2024-03-07T03:13:51.574122+0000 mon.a (mon.0) 917 : cluster 0 osdmap e109: 8 total, 8 up, 8 in 2024-03-07T03:13:54.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:53 smithi148 bash[21362]: cluster 2024-03-07T03:13:52.582283+0000 mon.a (mon.0) 918 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2024-03-07T03:13:54.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:53 smithi148 bash[31149]: cluster 2024-03-07T03:13:52.582283+0000 mon.a (mon.0) 918 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2024-03-07T03:13:54.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:53 smithi195 bash[23221]: cluster 2024-03-07T03:13:52.582283+0000 mon.a (mon.0) 918 : cluster 0 osdmap e110: 8 total, 8 up, 8 in 2024-03-07T03:13:54.649 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:13:54.667 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:13:54.889 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:13:55.017 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:13:55.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:54 smithi148 bash[21362]: cluster 2024-03-07T03:13:53.069893+0000 mgr.y (mgr.24287) 198 : cluster 0 pgmap v205: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:55.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:54 smithi148 bash[21362]: cluster 2024-03-07T03:13:53.589138+0000 mon.a (mon.0) 919 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2024-03-07T03:13:55.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:13:54 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:13:54] "GET /metrics HTTP/1.1" 200 36272 "" "Prometheus/2.43.0" 2024-03-07T03:13:55.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:54 smithi148 bash[31149]: cluster 2024-03-07T03:13:53.069893+0000 mgr.y (mgr.24287) 198 : cluster 0 pgmap v205: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:55.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:54 smithi148 bash[31149]: cluster 2024-03-07T03:13:53.589138+0000 mon.a (mon.0) 919 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2024-03-07T03:13:55.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:54 smithi195 bash[23221]: cluster 2024-03-07T03:13:53.069893+0000 mgr.y (mgr.24287) 198 : cluster 0 pgmap v205: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:13:55.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:54 smithi195 bash[23221]: cluster 2024-03-07T03:13:53.589138+0000 mon.a (mon.0) 919 : cluster 0 osdmap e111: 8 total, 8 up, 8 in 2024-03-07T03:13:55.143 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.Empty (2573 ms) 2024-03-07T03:13:55.143 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.OtherName 2024-03-07T03:13:55.197 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:13:55.217 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:13:55.431 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:13:55.539 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:13:55.664 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.OtherName (520 ms) 2024-03-07T03:13:55.665 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.OtherPool 2024-03-07T03:13:55.708 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:13:55.725 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:13:55.875 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:13:56.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:55 smithi148 bash[21362]: cluster 2024-03-07T03:13:54.600002+0000 mon.a (mon.0) 920 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2024-03-07T03:13:56.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:55 smithi148 bash[31149]: cluster 2024-03-07T03:13:54.600002+0000 mon.a (mon.0) 920 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2024-03-07T03:13:56.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:55 smithi195 bash[23221]: cluster 2024-03-07T03:13:54.600002+0000 mon.a (mon.0) 920 : cluster 0 osdmap e112: 8 total, 8 up, 8 in 2024-03-07T03:13:57.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:56 smithi148 bash[21362]: cluster 2024-03-07T03:13:55.070698+0000 mgr.y (mgr.24287) 199 : cluster 0 pgmap v208: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 8.5 KiB/s wr, 167 op/s 2024-03-07T03:13:57.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:56 smithi148 bash[31149]: cluster 2024-03-07T03:13:55.070698+0000 mgr.y (mgr.24287) 199 : cluster 0 pgmap v208: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 8.5 KiB/s wr, 167 op/s 2024-03-07T03:13:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:56 smithi195 bash[23221]: cluster 2024-03-07T03:13:55.070698+0000 mgr.y (mgr.24287) 199 : cluster 0 pgmap v208: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 8.5 KiB/s wr, 167 op/s 2024-03-07T03:13:57.782 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:13:57.907 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:13:58.033 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.OtherPool (2368 ms) 2024-03-07T03:13:58.033 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.OtherNamespace 2024-03-07T03:13:58.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:57 smithi148 bash[21362]: cluster 2024-03-07T03:13:56.607471+0000 mon.a (mon.0) 921 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2024-03-07T03:13:58.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:57 smithi148 bash[31149]: cluster 2024-03-07T03:13:56.607471+0000 mon.a (mon.0) 921 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2024-03-07T03:13:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:57 smithi195 bash[23221]: cluster 2024-03-07T03:13:56.607471+0000 mon.a (mon.0) 921 : cluster 0 osdmap e113: 8 total, 8 up, 8 in 2024-03-07T03:13:58.082 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:13:58.107 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:13:58.372 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:13:58.535 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:13:58.725 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.OtherNamespace (692 ms) 2024-03-07T03:13:58.726 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.DataPool 2024-03-07T03:13:58.804 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:13:58.831 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:13:59.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:58 smithi148 bash[31149]: cluster 2024-03-07T03:13:57.071138+0000 mgr.y (mgr.24287) 200 : cluster 0 pgmap v210: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 7.6 KiB/s wr, 148 op/s 2024-03-07T03:13:59.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:58 smithi148 bash[31149]: cluster 2024-03-07T03:13:57.622888+0000 mon.a (mon.0) 922 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2024-03-07T03:13:59.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:13:58 smithi148 bash[31149]: cluster 2024-03-07T03:13:58.291333+0000 mon.a (mon.0) 923 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2024-03-07T03:13:59.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:58 smithi148 bash[21362]: cluster 2024-03-07T03:13:57.071138+0000 mgr.y (mgr.24287) 200 : cluster 0 pgmap v210: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 7.6 KiB/s wr, 148 op/s 2024-03-07T03:13:59.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:58 smithi148 bash[21362]: cluster 2024-03-07T03:13:57.622888+0000 mon.a (mon.0) 922 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2024-03-07T03:13:59.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:13:58 smithi148 bash[21362]: cluster 2024-03-07T03:13:58.291333+0000 mon.a (mon.0) 923 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2024-03-07T03:13:59.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:58 smithi195 bash[23221]: cluster 2024-03-07T03:13:57.071138+0000 mgr.y (mgr.24287) 200 : cluster 0 pgmap v210: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 7.6 KiB/s wr, 148 op/s 2024-03-07T03:13:59.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:58 smithi195 bash[23221]: cluster 2024-03-07T03:13:57.622888+0000 mon.a (mon.0) 922 : cluster 0 osdmap e114: 8 total, 8 up, 8 in 2024-03-07T03:13:59.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:13:58 smithi195 bash[23221]: cluster 2024-03-07T03:13:58.291333+0000 mon.a (mon.0) 923 : cluster 0 osdmap e115: 8 total, 8 up, 8 in 2024-03-07T03:13:59.199 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:13:59.394 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:13:59.597 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.DataPool (872 ms) 2024-03-07T03:13:59.597 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-03-07T03:13:59.681 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:13:59.711 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:00.060 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:14:00.299 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.AbortAfterPrepare (702 ms) 2024-03-07T03:14:00.299 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-03-07T03:14:00.394 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:00.423 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:00.678 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (379 ms) 2024-03-07T03:14:00.678 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-03-07T03:14:00.757 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:00.784 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:01.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:00 smithi148 bash[21362]: cluster 2024-03-07T03:13:59.071811+0000 mgr.y (mgr.24287) 201 : cluster 0 pgmap v213: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 7.6 KiB/s wr, 149 op/s 2024-03-07T03:14:01.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:00 smithi148 bash[31149]: cluster 2024-03-07T03:13:59.071811+0000 mgr.y (mgr.24287) 201 : cluster 0 pgmap v213: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 7.6 KiB/s wr, 149 op/s 2024-03-07T03:14:01.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:00 smithi195 bash[23221]: cluster 2024-03-07T03:13:59.071811+0000 mgr.y (mgr.24287) 201 : cluster 0 pgmap v213: 137 pgs: 137 active+clean; 579 KiB data, 243 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 7.6 KiB/s wr, 149 op/s 2024-03-07T03:14:01.144 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:01.333 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:14:01.580 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.AbortAfterExecute (902 ms) 2024-03-07T03:14:01.580 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-03-07T03:14:01.665 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:01.693 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:02.045 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:02.234 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:14:02.482 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (902 ms) 2024-03-07T03:14:02.482 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-03-07T03:14:02.558 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:02.590 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:02.921 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:03.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:02 smithi148 bash[21362]: cluster 2024-03-07T03:14:01.073515+0000 mgr.y (mgr.24287) 202 : cluster 0 pgmap v214: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 504 KiB/s rd, 27 KiB/s wr, 631 op/s 2024-03-07T03:14:03.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:02 smithi148 bash[31149]: cluster 2024-03-07T03:14:01.073515+0000 mgr.y (mgr.24287) 202 : cluster 0 pgmap v214: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 504 KiB/s rd, 27 KiB/s wr, 631 op/s 2024-03-07T03:14:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:02 smithi195 bash[23221]: cluster 2024-03-07T03:14:01.073515+0000 mgr.y (mgr.24287) 202 : cluster 0 pgmap v214: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 504 KiB/s rd, 27 KiB/s wr, 631 op/s 2024-03-07T03:14:03.107 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:14:03.482 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (1000 ms) 2024-03-07T03:14:03.482 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-03-07T03:14:03.562 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:03.592 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_Migration.cc:635: Skipped 2024-03-07T03:14:03.592 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:14:03.608 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringSamePool (126 ms) 2024-03-07T03:14:03.608 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.MirroringAbort 2024-03-07T03:14:03.696 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:03.725 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_Migration.cc:653: Skipped 2024-03-07T03:14:03.725 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:14:03.742 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringAbort (135 ms) 2024-03-07T03:14:03.743 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-03-07T03:14:03.822 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:03.849 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_Migration.cc:676: Skipped 2024-03-07T03:14:03.849 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:14:03.867 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolDisabled (124 ms) 2024-03-07T03:14:03.867 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-03-07T03:14:03.944 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:03.974 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_Migration.cc:694: Skipped 2024-03-07T03:14:03.974 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:14:03.992 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolEnabled (125 ms) 2024-03-07T03:14:03.992 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.MirroringPool 2024-03-07T03:14:04.076 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:04.104 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_Migration.cc:714: Skipped 2024-03-07T03:14:04.104 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:14:04.121 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringPool (129 ms) 2024-03-07T03:14:04.121 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.Group 2024-03-07T03:14:04.197 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:04.249 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:04.538 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[31149]: cluster 2024-03-07T03:14:03.074305+0000 mgr.y (mgr.24287) 203 : cluster 0 pgmap v215: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 468 KiB/s rd, 25 KiB/s wr, 585 op/s 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[31149]: cluster 2024-03-07T03:14:03.288517+0000 mon.a (mon.0) 924 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[31149]: audit 2024-03-07T03:14:04.060186+0000 mon.b (mon.1) 128 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[31149]: audit 2024-03-07T03:14:04.060748+0000 mon.a (mon.0) 925 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[31149]: audit 2024-03-07T03:14:04.167412+0000 mon.a (mon.0) 926 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[21362]: cluster 2024-03-07T03:14:03.074305+0000 mgr.y (mgr.24287) 203 : cluster 0 pgmap v215: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 468 KiB/s rd, 25 KiB/s wr, 585 op/s 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[21362]: cluster 2024-03-07T03:14:03.288517+0000 mon.a (mon.0) 924 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[21362]: audit 2024-03-07T03:14:04.060186+0000 mon.b (mon.1) 128 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[21362]: audit 2024-03-07T03:14:04.060748+0000 mon.a (mon.0) 925 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[21362]: audit 2024-03-07T03:14:04.167412+0000 mon.a (mon.0) 926 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:04.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:04 smithi195 bash[23221]: cluster 2024-03-07T03:14:03.074305+0000 mgr.y (mgr.24287) 203 : cluster 0 pgmap v215: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 468 KiB/s rd, 25 KiB/s wr, 585 op/s 2024-03-07T03:14:04.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:04 smithi195 bash[23221]: cluster 2024-03-07T03:14:03.288517+0000 mon.a (mon.0) 924 : cluster 0 osdmap e116: 8 total, 8 up, 8 in 2024-03-07T03:14:04.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:04 smithi195 bash[23221]: audit 2024-03-07T03:14:04.060186+0000 mon.b (mon.1) 128 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:04.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:04 smithi195 bash[23221]: audit 2024-03-07T03:14:04.060748+0000 mon.a (mon.0) 925 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:04.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:04 smithi195 bash[23221]: audit 2024-03-07T03:14:04.167412+0000 mon.a (mon.0) 926 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:04.574 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:04.729 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:04.908 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.Group (787 ms) 2024-03-07T03:14:04.908 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.GroupAbort 2024-03-07T03:14:04.982 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:05.027 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:05.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:14:04 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:14:04] "GET /metrics HTTP/1.1" 200 38715 "" "Prometheus/2.43.0" 2024-03-07T03:14:05.325 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:14:05.560 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.GroupAbort (652 ms) 2024-03-07T03:14:05.560 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnaps 2024-03-07T03:14:05.622 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:05.645 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:05.740 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:05.818 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:06.145 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:06.331 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:06.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:06 smithi148 bash[21362]: cluster 2024-03-07T03:14:05.076026+0000 mgr.y (mgr.24287) 204 : cluster 0 pgmap v217: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 879 KiB/s rd, 57 KiB/s wr, 1.13k op/s 2024-03-07T03:14:06.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:06 smithi148 bash[31149]: cluster 2024-03-07T03:14:05.076026+0000 mgr.y (mgr.24287) 204 : cluster 0 pgmap v217: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 879 KiB/s rd, 57 KiB/s wr, 1.13k op/s 2024-03-07T03:14:06.544 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnaps (984 ms) 2024-03-07T03:14:06.544 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-03-07T03:14:06.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:06 smithi195 bash[23221]: cluster 2024-03-07T03:14:05.076026+0000 mgr.y (mgr.24287) 204 : cluster 0 pgmap v217: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 879 KiB/s rd, 57 KiB/s wr, 1.13k op/s 2024-03-07T03:14:06.616 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:06.641 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:06.726 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:06.801 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:06.818 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:06.879 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:07.169 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:07.300 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:07.483 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnapsOtherPool (939 ms) 2024-03-07T03:14:07.483 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-03-07T03:14:07.551 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:07.577 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:07.657 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:07.718 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:08.018 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:08.165 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:08.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: cluster 2024-03-07T03:14:07.076519+0000 mgr.y (mgr.24287) 205 : cluster 0 pgmap v218: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 745 KiB/s rd, 48 KiB/s wr, 959 op/s 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: audit 2024-03-07T03:14:07.156023+0000 mon.a (mon.0) 927 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: audit 2024-03-07T03:14:07.156144+0000 mon.b (mon.1) 129 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: cephadm 2024-03-07T03:14:07.157586+0000 mgr.y (mgr.24287) 206 : cephadm 1 Upgrade: Target is version 19.0.0-1905-gea9a5688 (unknown) 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: cephadm 2024-03-07T03:14:07.157675+0000 mgr.y (mgr.24287) 207 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71'] 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: audit 2024-03-07T03:14:07.169539+0000 mon.a (mon.0) 928 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: cephadm 2024-03-07T03:14:07.171284+0000 mgr.y (mgr.24287) 208 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[31149]: cephadm 2024-03-07T03:14:07.536502+0000 mgr.y (mgr.24287) 209 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 on smithi195 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: cluster 2024-03-07T03:14:07.076519+0000 mgr.y (mgr.24287) 205 : cluster 0 pgmap v218: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 745 KiB/s rd, 48 KiB/s wr, 959 op/s 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: audit 2024-03-07T03:14:07.156023+0000 mon.a (mon.0) 927 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: audit 2024-03-07T03:14:07.156144+0000 mon.b (mon.1) 129 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:08.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: cephadm 2024-03-07T03:14:07.157586+0000 mgr.y (mgr.24287) 206 : cephadm 1 Upgrade: Target is version 19.0.0-1905-gea9a5688 (unknown) 2024-03-07T03:14:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: cephadm 2024-03-07T03:14:07.157675+0000 mgr.y (mgr.24287) 207 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71'] 2024-03-07T03:14:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: audit 2024-03-07T03:14:07.169539+0000 mon.a (mon.0) 928 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: cephadm 2024-03-07T03:14:07.171284+0000 mgr.y (mgr.24287) 208 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:14:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:08 smithi148 bash[21362]: cephadm 2024-03-07T03:14:07.536502+0000 mgr.y (mgr.24287) 209 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 on smithi195 2024-03-07T03:14:08.327 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnapsDataPool (844 ms) 2024-03-07T03:14:08.327 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-03-07T03:14:08.372 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:08.391 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:08.460 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:08.513 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:08.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: cluster 2024-03-07T03:14:07.076519+0000 mgr.y (mgr.24287) 205 : cluster 0 pgmap v218: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 745 KiB/s rd, 48 KiB/s wr, 959 op/s 2024-03-07T03:14:08.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: audit 2024-03-07T03:14:07.156023+0000 mon.a (mon.0) 927 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:08.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: audit 2024-03-07T03:14:07.156144+0000 mon.b (mon.1) 129 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:08.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: cephadm 2024-03-07T03:14:07.157586+0000 mgr.y (mgr.24287) 206 : cephadm 1 Upgrade: Target is version 19.0.0-1905-gea9a5688 (unknown) 2024-03-07T03:14:08.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: cephadm 2024-03-07T03:14:07.157675+0000 mgr.y (mgr.24287) 207 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71, digests ['quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71'] 2024-03-07T03:14:08.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: audit 2024-03-07T03:14:07.169539+0000 mon.a (mon.0) 928 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:08.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: cephadm 2024-03-07T03:14:07.171284+0000 mgr.y (mgr.24287) 208 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:14:08.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:08 smithi195 bash[23221]: cephadm 2024-03-07T03:14:07.536502+0000 mgr.y (mgr.24287) 209 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 on smithi195 2024-03-07T03:14:08.847 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:14:08.869 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:09.112 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:09.328 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (1001 ms) 2024-03-07T03:14:09.328 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-03-07T03:14:09.405 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:09.434 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:09.527 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:09.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:09 smithi148 bash[21362]: audit 2024-03-07T03:14:08.483913+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.148:0/2693711324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:09.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:09 smithi148 bash[31149]: audit 2024-03-07T03:14:08.483913+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.148:0/2693711324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:09.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:09 smithi195 bash[23221]: audit 2024-03-07T03:14:08.483913+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.148:0/2693711324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:09.597 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 0 2024-03-07T03:14:09.630 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:09.899 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:10.075 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:10.199 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (871 ms) 2024-03-07T03:14:10.199 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-03-07T03:14:10.277 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:10.309 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:10.398 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:10.480 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:10.538 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:10 smithi148 bash[31149]: cluster 2024-03-07T03:14:09.077513+0000 mgr.y (mgr.24287) 210 : cluster 0 pgmap v219: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 655 KiB/s rd, 42 KiB/s wr, 843 op/s 2024-03-07T03:14:10.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:10 smithi148 bash[21362]: cluster 2024-03-07T03:14:09.077513+0000 mgr.y (mgr.24287) 210 : cluster 0 pgmap v219: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 655 KiB/s rd, 42 KiB/s wr, 843 op/s 2024-03-07T03:14:10.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:10 smithi195 bash[23221]: cluster 2024-03-07T03:14:09.077513+0000 mgr.y (mgr.24287) 210 : cluster 0 pgmap v219: 137 pgs: 137 active+clean; 581 KiB data, 256 MiB used, 715 GiB / 715 GiB avail; 655 KiB/s rd, 42 KiB/s wr, 843 op/s 2024-03-07T03:14:10.787 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 0 2024-03-07T03:14:10.814 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:10.938 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:11.022 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (824 ms) 2024-03-07T03:14:11.023 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-03-07T03:14:11.072 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:11.090 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:11.160 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:11.220 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:11.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:11 smithi195 bash[23221]: audit 2024-03-07T03:14:10.978309+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.148:0/2842409197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:11.583 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:14:11.602 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:11.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:11 smithi148 bash[21362]: audit 2024-03-07T03:14:10.978309+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.148:0/2842409197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:11 smithi148 bash[31149]: audit 2024-03-07T03:14:10.978309+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.148:0/2842409197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:11.819 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:12.100 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (1077 ms) 2024-03-07T03:14:12.100 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-03-07T03:14:12.179 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:12.212 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:12.295 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:12.364 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:12 smithi195 bash[23221]: cluster 2024-03-07T03:14:11.078474+0000 mgr.y (mgr.24287) 211 : cluster 0 pgmap v220: 137 pgs: 137 active+clean; 129 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s rd, 26 MiB/s wr, 1.05k op/s 2024-03-07T03:14:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:12 smithi148 bash[21362]: cluster 2024-03-07T03:14:11.078474+0000 mgr.y (mgr.24287) 211 : cluster 0 pgmap v220: 137 pgs: 137 active+clean; 129 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s rd, 26 MiB/s wr, 1.05k op/s 2024-03-07T03:14:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:12 smithi148 bash[31149]: cluster 2024-03-07T03:14:11.078474+0000 mgr.y (mgr.24287) 211 : cluster 0 pgmap v220: 137 pgs: 137 active+clean; 129 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s rd, 26 MiB/s wr, 1.05k op/s 2024-03-07T03:14:12.789 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: after_prepare_snap 2024-03-07T03:14:14.336 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:14:14.389 INFO:tasks.workunit.client.0.smithi148.stdout:write: * 0~1000 2024-03-07T03:14:14.403 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:14.619 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:14.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:14 smithi148 bash[21362]: cluster 2024-03-07T03:14:13.079261+0000 mgr.y (mgr.24287) 212 : cluster 0 pgmap v221: 137 pgs: 137 active+clean; 129 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s rd, 26 MiB/s wr, 1.05k op/s 2024-03-07T03:14:14.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:14 smithi148 bash[21362]: cluster 2024-03-07T03:14:13.319585+0000 mon.a (mon.0) 929 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2024-03-07T03:14:14.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:14 smithi148 bash[21362]: audit 2024-03-07T03:14:13.427295+0000 mon.a (mon.0) 930 : audit 0 from='client.? 172.21.15.148:0/256526298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:14.631 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:14 smithi148 bash[31149]: cluster 2024-03-07T03:14:13.079261+0000 mgr.y (mgr.24287) 212 : cluster 0 pgmap v221: 137 pgs: 137 active+clean; 129 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s rd, 26 MiB/s wr, 1.05k op/s 2024-03-07T03:14:14.631 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:14 smithi148 bash[31149]: cluster 2024-03-07T03:14:13.319585+0000 mon.a (mon.0) 929 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2024-03-07T03:14:14.631 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:14 smithi148 bash[31149]: audit 2024-03-07T03:14:13.427295+0000 mon.a (mon.0) 930 : audit 0 from='client.? 172.21.15.148:0/256526298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:14 smithi195 bash[23221]: cluster 2024-03-07T03:14:13.079261+0000 mgr.y (mgr.24287) 212 : cluster 0 pgmap v221: 137 pgs: 137 active+clean; 129 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 MiB/s rd, 26 MiB/s wr, 1.05k op/s 2024-03-07T03:14:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:14 smithi195 bash[23221]: cluster 2024-03-07T03:14:13.319585+0000 mon.a (mon.0) 929 : cluster 0 osdmap e117: 8 total, 8 up, 8 in 2024-03-07T03:14:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:14 smithi195 bash[23221]: audit 2024-03-07T03:14:13.427295+0000 mon.a (mon.0) 930 : audit 0 from='client.? 172.21.15.148:0/256526298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:14.898 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (2799 ms) 2024-03-07T03:14:14.899 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.Snaps 2024-03-07T03:14:14.983 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:15.011 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:15.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:14:14 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:14:14] "GET /metrics HTTP/1.1" 200 38715 "" "Prometheus/2.43.0" 2024-03-07T03:14:15.099 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:14:15.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:15 smithi148 bash[21362]: cluster 2024-03-07T03:14:14.331826+0000 mon.a (mon.0) 931 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2024-03-07T03:14:15.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:15 smithi148 bash[31149]: cluster 2024-03-07T03:14:14.331826+0000 mon.a (mon.0) 931 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2024-03-07T03:14:15.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:15 smithi195 bash[23221]: cluster 2024-03-07T03:14:14.331826+0000 mon.a (mon.0) 931 : cluster 0 osdmap e118: 8 total, 8 up, 8 in 2024-03-07T03:14:16.370 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:14:16.400 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:14:16.401 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:14:16.401 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:14:16.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:16 smithi148 bash[21362]: cluster 2024-03-07T03:14:15.081005+0000 mgr.y (mgr.24287) 213 : cluster 0 pgmap v224: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 69 MiB/s rd, 48 MiB/s wr, 1.24k op/s 2024-03-07T03:14:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:16 smithi148 bash[21362]: cluster 2024-03-07T03:14:15.356785+0000 mon.a (mon.0) 932 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2024-03-07T03:14:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:16 smithi148 bash[21362]: audit 2024-03-07T03:14:15.869960+0000 mon.a (mon.0) 933 : audit 0 from='client.? 172.21.15.148:0/877810126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:16 smithi148 bash[31149]: cluster 2024-03-07T03:14:15.081005+0000 mgr.y (mgr.24287) 213 : cluster 0 pgmap v224: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 69 MiB/s rd, 48 MiB/s wr, 1.24k op/s 2024-03-07T03:14:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:16 smithi148 bash[31149]: cluster 2024-03-07T03:14:15.356785+0000 mon.a (mon.0) 932 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2024-03-07T03:14:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:16 smithi148 bash[31149]: audit 2024-03-07T03:14:15.869960+0000 mon.a (mon.0) 933 : audit 0 from='client.? 172.21.15.148:0/877810126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:16 smithi195 bash[23221]: cluster 2024-03-07T03:14:15.081005+0000 mgr.y (mgr.24287) 213 : cluster 0 pgmap v224: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 69 MiB/s rd, 48 MiB/s wr, 1.24k op/s 2024-03-07T03:14:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:16 smithi195 bash[23221]: cluster 2024-03-07T03:14:15.356785+0000 mon.a (mon.0) 932 : cluster 0 osdmap e119: 8 total, 8 up, 8 in 2024-03-07T03:14:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:16 smithi195 bash[23221]: audit 2024-03-07T03:14:15.869960+0000 mon.a (mon.0) 933 : audit 0 from='client.? 172.21.15.148:0/877810126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:17.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:17 smithi148 bash[21362]: cluster 2024-03-07T03:14:16.358425+0000 mon.a (mon.0) 934 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2024-03-07T03:14:17.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:17 smithi148 bash[31149]: cluster 2024-03-07T03:14:16.358425+0000 mon.a (mon.0) 934 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2024-03-07T03:14:17.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:17 smithi195 bash[23221]: cluster 2024-03-07T03:14:16.358425+0000 mon.a (mon.0) 934 : cluster 0 osdmap e120: 8 total, 8 up, 8 in 2024-03-07T03:14:18.392 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:14:18.408 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:14:18.408 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:14:18.427 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:14:18.470 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:14:18.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:18 smithi148 bash[21362]: cluster 2024-03-07T03:14:17.081755+0000 mgr.y (mgr.24287) 214 : cluster 0 pgmap v227: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 57 MiB/s rd, 32 MiB/s wr, 1.01k op/s 2024-03-07T03:14:18.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:18 smithi148 bash[21362]: cluster 2024-03-07T03:14:17.379868+0000 mon.a (mon.0) 935 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2024-03-07T03:14:18.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:18 smithi148 bash[21362]: audit 2024-03-07T03:14:18.303180+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.148:0/243180176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:18.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:18 smithi148 bash[31149]: cluster 2024-03-07T03:14:17.081755+0000 mgr.y (mgr.24287) 214 : cluster 0 pgmap v227: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 57 MiB/s rd, 32 MiB/s wr, 1.01k op/s 2024-03-07T03:14:18.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:18 smithi148 bash[31149]: cluster 2024-03-07T03:14:17.379868+0000 mon.a (mon.0) 935 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2024-03-07T03:14:18.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:18 smithi148 bash[31149]: audit 2024-03-07T03:14:18.303180+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.148:0/243180176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:18 smithi195 bash[23221]: cluster 2024-03-07T03:14:17.081755+0000 mgr.y (mgr.24287) 214 : cluster 0 pgmap v227: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 57 MiB/s rd, 32 MiB/s wr, 1.01k op/s 2024-03-07T03:14:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:18 smithi195 bash[23221]: cluster 2024-03-07T03:14:17.379868+0000 mon.a (mon.0) 935 : cluster 0 osdmap e121: 8 total, 8 up, 8 in 2024-03-07T03:14:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:18 smithi195 bash[23221]: audit 2024-03-07T03:14:18.303180+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.148:0/243180176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:19.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:19 smithi148 bash[21362]: cluster 2024-03-07T03:14:18.386361+0000 mon.a (mon.0) 936 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2024-03-07T03:14:19.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:19 smithi148 bash[21362]: audit 2024-03-07T03:14:19.050520+0000 mon.b (mon.1) 130 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:19.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:19 smithi148 bash[31149]: cluster 2024-03-07T03:14:18.386361+0000 mon.a (mon.0) 936 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2024-03-07T03:14:19.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:19 smithi148 bash[31149]: audit 2024-03-07T03:14:19.050520+0000 mon.b (mon.1) 130 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:19 smithi195 bash[23221]: cluster 2024-03-07T03:14:18.386361+0000 mon.a (mon.0) 936 : cluster 0 osdmap e122: 8 total, 8 up, 8 in 2024-03-07T03:14:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:19 smithi195 bash[23221]: audit 2024-03-07T03:14:19.050520+0000 mon.b (mon.1) 130 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:20.418 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:20.464 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:14:20.478 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:20.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:20 smithi148 bash[31149]: cluster 2024-03-07T03:14:19.082405+0000 mgr.y (mgr.24287) 215 : cluster 0 pgmap v230: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:20.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:20 smithi148 bash[31149]: cluster 2024-03-07T03:14:19.406004+0000 mon.a (mon.0) 937 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2024-03-07T03:14:20.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:20 smithi148 bash[21362]: cluster 2024-03-07T03:14:19.082405+0000 mgr.y (mgr.24287) 215 : cluster 0 pgmap v230: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:20.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:20 smithi148 bash[21362]: cluster 2024-03-07T03:14:19.406004+0000 mon.a (mon.0) 937 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2024-03-07T03:14:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:20 smithi195 bash[23221]: cluster 2024-03-07T03:14:19.082405+0000 mgr.y (mgr.24287) 215 : cluster 0 pgmap v230: 137 pgs: 137 active+clean; 193 MiB data, 865 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:20 smithi195 bash[23221]: cluster 2024-03-07T03:14:19.406004+0000 mon.a (mon.0) 937 : cluster 0 osdmap e123: 8 total, 8 up, 8 in 2024-03-07T03:14:21.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:21 smithi148 bash[21362]: cluster 2024-03-07T03:14:20.409828+0000 mon.a (mon.0) 938 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2024-03-07T03:14:21.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:21 smithi148 bash[21362]: audit 2024-03-07T03:14:20.756050+0000 mon.a (mon.0) 939 : audit 0 from='client.? 172.21.15.148:0/2030108959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:21.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:21 smithi148 bash[31149]: cluster 2024-03-07T03:14:20.409828+0000 mon.a (mon.0) 938 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2024-03-07T03:14:21.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:21 smithi148 bash[31149]: audit 2024-03-07T03:14:20.756050+0000 mon.a (mon.0) 939 : audit 0 from='client.? 172.21.15.148:0/2030108959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:21 smithi195 bash[23221]: cluster 2024-03-07T03:14:20.409828+0000 mon.a (mon.0) 938 : cluster 0 osdmap e124: 8 total, 8 up, 8 in 2024-03-07T03:14:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:21 smithi195 bash[23221]: audit 2024-03-07T03:14:20.756050+0000 mon.a (mon.0) 939 : audit 0 from='client.? 172.21.15.148:0/2030108959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:22.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:22 smithi148 bash[21362]: cluster 2024-03-07T03:14:21.099834+0000 mgr.y (mgr.24287) 216 : cluster 0 pgmap v233: 137 pgs: 137 active+clean; 249 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:14:22.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:22 smithi148 bash[21362]: cluster 2024-03-07T03:14:21.421868+0000 mon.a (mon.0) 940 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2024-03-07T03:14:22.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:22 smithi148 bash[31149]: cluster 2024-03-07T03:14:21.099834+0000 mgr.y (mgr.24287) 216 : cluster 0 pgmap v233: 137 pgs: 137 active+clean; 249 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:14:22.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:22 smithi148 bash[31149]: cluster 2024-03-07T03:14:21.421868+0000 mon.a (mon.0) 940 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2024-03-07T03:14:22.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:22 smithi195 bash[23221]: cluster 2024-03-07T03:14:21.099834+0000 mgr.y (mgr.24287) 216 : cluster 0 pgmap v233: 137 pgs: 137 active+clean; 249 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:14:22.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:22 smithi195 bash[23221]: cluster 2024-03-07T03:14:21.421868+0000 mon.a (mon.0) 940 : cluster 0 osdmap e125: 8 total, 8 up, 8 in 2024-03-07T03:14:23.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:23 smithi148 bash[21362]: cluster 2024-03-07T03:14:22.423427+0000 mon.a (mon.0) 941 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2024-03-07T03:14:23.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:23 smithi148 bash[21362]: audit 2024-03-07T03:14:23.191088+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.148:0/4279563898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:23 smithi148 bash[31149]: cluster 2024-03-07T03:14:22.423427+0000 mon.a (mon.0) 941 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2024-03-07T03:14:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:23 smithi148 bash[31149]: audit 2024-03-07T03:14:23.191088+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.148:0/4279563898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:23.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:23 smithi195 bash[23221]: cluster 2024-03-07T03:14:22.423427+0000 mon.a (mon.0) 941 : cluster 0 osdmap e126: 8 total, 8 up, 8 in 2024-03-07T03:14:23.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:23 smithi195 bash[23221]: audit 2024-03-07T03:14:23.191088+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.148:0/4279563898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:24.332 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:24.596 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:24.788 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:14:24 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:14:24] "GET /metrics HTTP/1.1" 200 38766 "" "Prometheus/2.43.0" 2024-03-07T03:14:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:24 smithi148 bash[21362]: cluster 2024-03-07T03:14:23.100536+0000 mgr.y (mgr.24287) 217 : cluster 0 pgmap v236: 137 pgs: 137 active+clean; 249 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:14:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:24 smithi148 bash[21362]: cluster 2024-03-07T03:14:23.449344+0000 mon.a (mon.0) 942 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2024-03-07T03:14:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:24 smithi148 bash[31149]: cluster 2024-03-07T03:14:23.100536+0000 mgr.y (mgr.24287) 217 : cluster 0 pgmap v236: 137 pgs: 137 active+clean; 249 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:14:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:24 smithi148 bash[31149]: cluster 2024-03-07T03:14:23.449344+0000 mon.a (mon.0) 942 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2024-03-07T03:14:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:24 smithi195 bash[23221]: cluster 2024-03-07T03:14:23.100536+0000 mgr.y (mgr.24287) 217 : cluster 0 pgmap v236: 137 pgs: 137 active+clean; 249 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 12 MiB/s wr, 124 op/s 2024-03-07T03:14:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:24 smithi195 bash[23221]: cluster 2024-03-07T03:14:23.449344+0000 mon.a (mon.0) 942 : cluster 0 osdmap e127: 8 total, 8 up, 8 in 2024-03-07T03:14:26.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:26 smithi148 bash[31149]: cluster 2024-03-07T03:14:25.102085+0000 mgr.y (mgr.24287) 218 : cluster 0 pgmap v238: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 5.1 MiB/s wr, 419 op/s 2024-03-07T03:14:26.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:26 smithi148 bash[31149]: cluster 2024-03-07T03:14:25.458308+0000 mon.a (mon.0) 943 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2024-03-07T03:14:26.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:26 smithi148 bash[31149]: audit 2024-03-07T03:14:25.647775+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.148:0/3300290684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:26.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:26 smithi148 bash[21362]: cluster 2024-03-07T03:14:25.102085+0000 mgr.y (mgr.24287) 218 : cluster 0 pgmap v238: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 5.1 MiB/s wr, 419 op/s 2024-03-07T03:14:26.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:26 smithi148 bash[21362]: cluster 2024-03-07T03:14:25.458308+0000 mon.a (mon.0) 943 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2024-03-07T03:14:26.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:26 smithi148 bash[21362]: audit 2024-03-07T03:14:25.647775+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.148:0/3300290684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:26.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:26 smithi195 bash[23221]: cluster 2024-03-07T03:14:25.102085+0000 mgr.y (mgr.24287) 218 : cluster 0 pgmap v238: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 5.1 MiB/s wr, 419 op/s 2024-03-07T03:14:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:26 smithi195 bash[23221]: cluster 2024-03-07T03:14:25.458308+0000 mon.a (mon.0) 943 : cluster 0 osdmap e128: 8 total, 8 up, 8 in 2024-03-07T03:14:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:26 smithi195 bash[23221]: audit 2024-03-07T03:14:25.647775+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.148:0/3300290684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:27.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:27 smithi148 bash[21362]: cluster 2024-03-07T03:14:26.470551+0000 mon.a (mon.0) 944 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2024-03-07T03:14:27.845 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:27 smithi148 bash[31149]: cluster 2024-03-07T03:14:26.470551+0000 mon.a (mon.0) 944 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2024-03-07T03:14:27.846 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:27 smithi195 bash[23221]: cluster 2024-03-07T03:14:26.470551+0000 mon.a (mon.0) 944 : cluster 0 osdmap e129: 8 total, 8 up, 8 in 2024-03-07T03:14:27.935 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.Snaps (13036 ms) 2024-03-07T03:14:27.935 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-03-07T03:14:27.984 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:28.001 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:28.090 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:14:28.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:28 smithi148 bash[21362]: cluster 2024-03-07T03:14:27.102849+0000 mgr.y (mgr.24287) 219 : cluster 0 pgmap v241: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 5.1 MiB/s wr, 420 op/s 2024-03-07T03:14:28.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:28 smithi148 bash[21362]: cluster 2024-03-07T03:14:27.484044+0000 mon.a (mon.0) 945 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2024-03-07T03:14:28.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:28 smithi148 bash[21362]: audit 2024-03-07T03:14:28.142935+0000 mon.a (mon.0) 946 : audit 0 from='client.? 172.21.15.148:0/1729032736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:28.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:28 smithi148 bash[31149]: cluster 2024-03-07T03:14:27.102849+0000 mgr.y (mgr.24287) 219 : cluster 0 pgmap v241: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 5.1 MiB/s wr, 420 op/s 2024-03-07T03:14:28.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:28 smithi148 bash[31149]: cluster 2024-03-07T03:14:27.484044+0000 mon.a (mon.0) 945 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2024-03-07T03:14:28.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:28 smithi148 bash[31149]: audit 2024-03-07T03:14:28.142935+0000 mon.a (mon.0) 946 : audit 0 from='client.? 172.21.15.148:0/1729032736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:28.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:28 smithi195 bash[23221]: cluster 2024-03-07T03:14:27.102849+0000 mgr.y (mgr.24287) 219 : cluster 0 pgmap v241: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 33 MiB/s rd, 5.1 MiB/s wr, 420 op/s 2024-03-07T03:14:28.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:28 smithi195 bash[23221]: cluster 2024-03-07T03:14:27.484044+0000 mon.a (mon.0) 945 : cluster 0 osdmap e130: 8 total, 8 up, 8 in 2024-03-07T03:14:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:28 smithi195 bash[23221]: audit 2024-03-07T03:14:28.142935+0000 mon.a (mon.0) 946 : audit 0 from='client.? 172.21.15.148:0/1729032736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:29.506 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:14:29.530 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:14:29.531 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:14:29.531 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:14:29.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:29 smithi148 bash[31149]: cluster 2024-03-07T03:14:28.482748+0000 mon.a (mon.0) 947 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2024-03-07T03:14:29.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:29 smithi148 bash[21362]: cluster 2024-03-07T03:14:28.482748+0000 mon.a (mon.0) 947 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2024-03-07T03:14:29.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:29 smithi195 bash[23221]: cluster 2024-03-07T03:14:28.482748+0000 mon.a (mon.0) 947 : cluster 0 osdmap e131: 8 total, 8 up, 8 in 2024-03-07T03:14:30.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:30 smithi148 bash[21362]: cluster 2024-03-07T03:14:29.103575+0000 mgr.y (mgr.24287) 220 : cluster 0 pgmap v244: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:30.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:30 smithi148 bash[21362]: cluster 2024-03-07T03:14:29.504862+0000 mon.a (mon.0) 948 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2024-03-07T03:14:30.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:30 smithi148 bash[31149]: cluster 2024-03-07T03:14:29.103575+0000 mgr.y (mgr.24287) 220 : cluster 0 pgmap v244: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:30.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:30 smithi148 bash[31149]: cluster 2024-03-07T03:14:29.504862+0000 mon.a (mon.0) 948 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2024-03-07T03:14:30.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:30 smithi195 bash[23221]: cluster 2024-03-07T03:14:29.103575+0000 mgr.y (mgr.24287) 220 : cluster 0 pgmap v244: 137 pgs: 137 active+clean; 277 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:30.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:30 smithi195 bash[23221]: cluster 2024-03-07T03:14:29.504862+0000 mon.a (mon.0) 948 : cluster 0 osdmap e132: 8 total, 8 up, 8 in 2024-03-07T03:14:31.527 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:14:31.540 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:14:31.540 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:14:31.558 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:14:31.604 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:14:31.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:31 smithi148 bash[21362]: cluster 2024-03-07T03:14:30.515558+0000 mon.a (mon.0) 949 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2024-03-07T03:14:31.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:31 smithi148 bash[21362]: audit 2024-03-07T03:14:30.591084+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.148:0/376770646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:31.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:31 smithi148 bash[31149]: cluster 2024-03-07T03:14:30.515558+0000 mon.a (mon.0) 949 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2024-03-07T03:14:31.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:31 smithi148 bash[31149]: audit 2024-03-07T03:14:30.591084+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.148:0/376770646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:31.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:31 smithi195 bash[23221]: cluster 2024-03-07T03:14:30.515558+0000 mon.a (mon.0) 949 : cluster 0 osdmap e133: 8 total, 8 up, 8 in 2024-03-07T03:14:31.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:31 smithi195 bash[23221]: audit 2024-03-07T03:14:30.591084+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.148:0/376770646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:32.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:32 smithi148 bash[21362]: cluster 2024-03-07T03:14:31.104385+0000 mgr.y (mgr.24287) 221 : cluster 0 pgmap v247: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-03-07T03:14:32.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:32 smithi148 bash[21362]: cluster 2024-03-07T03:14:31.525170+0000 mon.a (mon.0) 950 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2024-03-07T03:14:32.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:32 smithi148 bash[31149]: cluster 2024-03-07T03:14:31.104385+0000 mgr.y (mgr.24287) 221 : cluster 0 pgmap v247: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-03-07T03:14:32.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:32 smithi148 bash[31149]: cluster 2024-03-07T03:14:31.525170+0000 mon.a (mon.0) 950 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2024-03-07T03:14:32.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:32 smithi195 bash[23221]: cluster 2024-03-07T03:14:31.104385+0000 mgr.y (mgr.24287) 221 : cluster 0 pgmap v247: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-03-07T03:14:32.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:32 smithi195 bash[23221]: cluster 2024-03-07T03:14:31.525170+0000 mon.a (mon.0) 950 : cluster 0 osdmap e134: 8 total, 8 up, 8 in 2024-03-07T03:14:33.310 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:33.347 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:14:33.361 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:33.396 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:14:33.457 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:33.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:33 smithi148 bash[21362]: cluster 2024-03-07T03:14:32.527840+0000 mon.a (mon.0) 951 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2024-03-07T03:14:33.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:33 smithi148 bash[21362]: audit 2024-03-07T03:14:33.028476+0000 mon.a (mon.0) 952 : audit 0 from='client.? 172.21.15.148:0/1252145530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:33.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:33 smithi148 bash[21362]: cluster 2024-03-07T03:14:33.299555+0000 mon.a (mon.0) 953 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2024-03-07T03:14:33.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:33 smithi148 bash[31149]: cluster 2024-03-07T03:14:32.527840+0000 mon.a (mon.0) 951 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2024-03-07T03:14:33.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:33 smithi148 bash[31149]: audit 2024-03-07T03:14:33.028476+0000 mon.a (mon.0) 952 : audit 0 from='client.? 172.21.15.148:0/1252145530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:33.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:33 smithi148 bash[31149]: cluster 2024-03-07T03:14:33.299555+0000 mon.a (mon.0) 953 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2024-03-07T03:14:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:33 smithi195 bash[23221]: cluster 2024-03-07T03:14:32.527840+0000 mon.a (mon.0) 951 : cluster 0 osdmap e135: 8 total, 8 up, 8 in 2024-03-07T03:14:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:33 smithi195 bash[23221]: audit 2024-03-07T03:14:33.028476+0000 mon.a (mon.0) 952 : audit 0 from='client.? 172.21.15.148:0/1252145530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:33.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:33 smithi195 bash[23221]: cluster 2024-03-07T03:14:33.299555+0000 mon.a (mon.0) 953 : cluster 0 osdmap e136: 8 total, 8 up, 8 in 2024-03-07T03:14:34.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:34 smithi195 bash[23221]: cluster 2024-03-07T03:14:33.104787+0000 mgr.y (mgr.24287) 222 : cluster 0 pgmap v250: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-03-07T03:14:34.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:34 smithi195 bash[23221]: audit 2024-03-07T03:14:34.050632+0000 mon.b (mon.1) 131 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:35.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:14:34 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:14:34] "GET /metrics HTTP/1.1" 200 38770 "" "Prometheus/2.43.0" 2024-03-07T03:14:35.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:34 smithi148 bash[21362]: cluster 2024-03-07T03:14:33.104787+0000 mgr.y (mgr.24287) 222 : cluster 0 pgmap v250: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-03-07T03:14:35.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:34 smithi148 bash[21362]: audit 2024-03-07T03:14:34.050632+0000 mon.b (mon.1) 131 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:35.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:34 smithi148 bash[31149]: cluster 2024-03-07T03:14:33.104787+0000 mgr.y (mgr.24287) 222 : cluster 0 pgmap v250: 137 pgs: 137 active+clean; 281 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 8.0 MiB/s wr, 144 op/s 2024-03-07T03:14:35.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:34 smithi148 bash[31149]: audit 2024-03-07T03:14:34.050632+0000 mon.b (mon.1) 131 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:35 smithi195 bash[23221]: cluster 2024-03-07T03:14:34.547216+0000 mon.a (mon.0) 954 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2024-03-07T03:14:35.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:35 smithi195 bash[23221]: audit 2024-03-07T03:14:35.462043+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.148:0/3846951123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:36.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:35 smithi148 bash[21362]: cluster 2024-03-07T03:14:34.547216+0000 mon.a (mon.0) 954 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2024-03-07T03:14:36.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:35 smithi148 bash[21362]: audit 2024-03-07T03:14:35.462043+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.148:0/3846951123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:36.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:35 smithi148 bash[31149]: cluster 2024-03-07T03:14:34.547216+0000 mon.a (mon.0) 954 : cluster 0 osdmap e137: 8 total, 8 up, 8 in 2024-03-07T03:14:36.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:35 smithi148 bash[31149]: audit 2024-03-07T03:14:35.462043+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.148:0/3846951123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:37.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:36 smithi148 bash[21362]: cluster 2024-03-07T03:14:35.106342+0000 mgr.y (mgr.24287) 223 : cluster 0 pgmap v253: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-03-07T03:14:37.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:36 smithi148 bash[21362]: cluster 2024-03-07T03:14:35.573590+0000 mon.a (mon.0) 955 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2024-03-07T03:14:37.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:36 smithi148 bash[31149]: cluster 2024-03-07T03:14:35.106342+0000 mgr.y (mgr.24287) 223 : cluster 0 pgmap v253: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-03-07T03:14:37.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:36 smithi148 bash[31149]: cluster 2024-03-07T03:14:35.573590+0000 mon.a (mon.0) 955 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2024-03-07T03:14:37.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:36 smithi195 bash[23221]: cluster 2024-03-07T03:14:35.106342+0000 mgr.y (mgr.24287) 223 : cluster 0 pgmap v253: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-03-07T03:14:37.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:36 smithi195 bash[23221]: cluster 2024-03-07T03:14:35.573590+0000 mon.a (mon.0) 955 : cluster 0 osdmap e138: 8 total, 8 up, 8 in 2024-03-07T03:14:37.415 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:37.800 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:37 smithi195 bash[23221]: cluster 2024-03-07T03:14:36.575242+0000 mon.a (mon.0) 956 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2024-03-07T03:14:38.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:37 smithi148 bash[21362]: cluster 2024-03-07T03:14:36.575242+0000 mon.a (mon.0) 956 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2024-03-07T03:14:38.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:37 smithi148 bash[31149]: cluster 2024-03-07T03:14:36.575242+0000 mon.a (mon.0) 956 : cluster 0 osdmap e139: 8 total, 8 up, 8 in 2024-03-07T03:14:39.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:38 smithi148 bash[21362]: cluster 2024-03-07T03:14:37.107376+0000 mgr.y (mgr.24287) 224 : cluster 0 pgmap v256: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-03-07T03:14:39.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:38 smithi148 bash[21362]: audit 2024-03-07T03:14:37.946959+0000 mon.a (mon.0) 957 : audit 0 from='client.? 172.21.15.148:0/2667192919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:39.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:38 smithi148 bash[31149]: cluster 2024-03-07T03:14:37.107376+0000 mgr.y (mgr.24287) 224 : cluster 0 pgmap v256: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-03-07T03:14:39.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:38 smithi148 bash[31149]: audit 2024-03-07T03:14:37.946959+0000 mon.a (mon.0) 957 : audit 0 from='client.? 172.21.15.148:0/2667192919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:39.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:38 smithi195 bash[23221]: cluster 2024-03-07T03:14:37.107376+0000 mgr.y (mgr.24287) 224 : cluster 0 pgmap v256: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 MiB/s wr, 197 op/s 2024-03-07T03:14:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:38 smithi195 bash[23221]: audit 2024-03-07T03:14:37.946959+0000 mon.a (mon.0) 957 : audit 0 from='client.? 172.21.15.148:0/2667192919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:40.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:39 smithi148 bash[21362]: cluster 2024-03-07T03:14:38.626818+0000 mon.a (mon.0) 958 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2024-03-07T03:14:40.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:39 smithi148 bash[31149]: cluster 2024-03-07T03:14:38.626818+0000 mon.a (mon.0) 958 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2024-03-07T03:14:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:39 smithi195 bash[23221]: cluster 2024-03-07T03:14:38.626818+0000 mon.a (mon.0) 958 : cluster 0 osdmap e140: 8 total, 8 up, 8 in 2024-03-07T03:14:41.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:40 smithi148 bash[21362]: cluster 2024-03-07T03:14:39.108131+0000 mgr.y (mgr.24287) 225 : cluster 0 pgmap v258: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 11 MiB/s wr, 172 op/s 2024-03-07T03:14:41.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:40 smithi148 bash[21362]: cluster 2024-03-07T03:14:39.635279+0000 mon.a (mon.0) 959 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2024-03-07T03:14:41.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:40 smithi148 bash[21362]: audit 2024-03-07T03:14:40.390201+0000 mon.a (mon.0) 960 : audit 0 from='client.? 172.21.15.148:0/2252526994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:41.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:40 smithi148 bash[31149]: cluster 2024-03-07T03:14:39.108131+0000 mgr.y (mgr.24287) 225 : cluster 0 pgmap v258: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 11 MiB/s wr, 172 op/s 2024-03-07T03:14:41.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:40 smithi148 bash[31149]: cluster 2024-03-07T03:14:39.635279+0000 mon.a (mon.0) 959 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2024-03-07T03:14:41.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:40 smithi148 bash[31149]: audit 2024-03-07T03:14:40.390201+0000 mon.a (mon.0) 960 : audit 0 from='client.? 172.21.15.148:0/2252526994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:41.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:40 smithi195 bash[23221]: cluster 2024-03-07T03:14:39.108131+0000 mgr.y (mgr.24287) 225 : cluster 0 pgmap v258: 137 pgs: 137 active+clean; 337 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 11 MiB/s wr, 172 op/s 2024-03-07T03:14:41.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:40 smithi195 bash[23221]: cluster 2024-03-07T03:14:39.635279+0000 mon.a (mon.0) 959 : cluster 0 osdmap e141: 8 total, 8 up, 8 in 2024-03-07T03:14:41.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:40 smithi195 bash[23221]: audit 2024-03-07T03:14:40.390201+0000 mon.a (mon.0) 960 : audit 0 from='client.? 172.21.15.148:0/2252526994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:41.231 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsOtherPool (13297 ms) 2024-03-07T03:14:41.231 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-03-07T03:14:41.322 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:41.353 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:41.471 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:14:42.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:41 smithi148 bash[21362]: cluster 2024-03-07T03:14:40.636587+0000 mon.a (mon.0) 961 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2024-03-07T03:14:42.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:41 smithi148 bash[31149]: cluster 2024-03-07T03:14:40.636587+0000 mon.a (mon.0) 961 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2024-03-07T03:14:42.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:41 smithi195 bash[23221]: cluster 2024-03-07T03:14:40.636587+0000 mon.a (mon.0) 961 : cluster 0 osdmap e142: 8 total, 8 up, 8 in 2024-03-07T03:14:42.679 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:14:42.710 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:14:42.711 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:14:42.711 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:14:43.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:42 smithi148 bash[31149]: cluster 2024-03-07T03:14:41.109037+0000 mgr.y (mgr.24287) 226 : cluster 0 pgmap v261: 137 pgs: 137 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s rd, 8.8 MiB/s wr, 331 op/s 2024-03-07T03:14:43.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:42 smithi148 bash[31149]: cluster 2024-03-07T03:14:41.661807+0000 mon.a (mon.0) 962 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2024-03-07T03:14:43.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:42 smithi148 bash[21362]: cluster 2024-03-07T03:14:41.109037+0000 mgr.y (mgr.24287) 226 : cluster 0 pgmap v261: 137 pgs: 137 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s rd, 8.8 MiB/s wr, 331 op/s 2024-03-07T03:14:43.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:42 smithi148 bash[21362]: cluster 2024-03-07T03:14:41.661807+0000 mon.a (mon.0) 962 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2024-03-07T03:14:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:42 smithi195 bash[23221]: cluster 2024-03-07T03:14:41.109037+0000 mgr.y (mgr.24287) 226 : cluster 0 pgmap v261: 137 pgs: 137 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 41 MiB/s rd, 8.8 MiB/s wr, 331 op/s 2024-03-07T03:14:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:42 smithi195 bash[23221]: cluster 2024-03-07T03:14:41.661807+0000 mon.a (mon.0) 962 : cluster 0 osdmap e143: 8 total, 8 up, 8 in 2024-03-07T03:14:44.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:43 smithi148 bash[21362]: cluster 2024-03-07T03:14:42.664704+0000 mon.a (mon.0) 963 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2024-03-07T03:14:44.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:43 smithi148 bash[21362]: audit 2024-03-07T03:14:42.844535+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.148:0/767398448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:44.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:43 smithi148 bash[21362]: cluster 2024-03-07T03:14:43.303937+0000 mon.a (mon.0) 964 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2024-03-07T03:14:44.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:43 smithi148 bash[31149]: cluster 2024-03-07T03:14:42.664704+0000 mon.a (mon.0) 963 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2024-03-07T03:14:44.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:43 smithi148 bash[31149]: audit 2024-03-07T03:14:42.844535+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.148:0/767398448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:44.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:43 smithi148 bash[31149]: cluster 2024-03-07T03:14:43.303937+0000 mon.a (mon.0) 964 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2024-03-07T03:14:44.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:43 smithi195 bash[23221]: cluster 2024-03-07T03:14:42.664704+0000 mon.a (mon.0) 963 : cluster 0 osdmap e144: 8 total, 8 up, 8 in 2024-03-07T03:14:44.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:43 smithi195 bash[23221]: audit 2024-03-07T03:14:42.844535+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.148:0/767398448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:44.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:43 smithi195 bash[23221]: cluster 2024-03-07T03:14:43.303937+0000 mon.a (mon.0) 964 : cluster 0 osdmap e145: 8 total, 8 up, 8 in 2024-03-07T03:14:44.330 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:14:44.344 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:14:44.345 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:14:44.370 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:14:44.420 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:14:45.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:14:44 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:14:44] "GET /metrics HTTP/1.1" 200 38770 "" "Prometheus/2.43.0" 2024-03-07T03:14:45.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:44 smithi148 bash[21362]: cluster 2024-03-07T03:14:43.109802+0000 mgr.y (mgr.24287) 227 : cluster 0 pgmap v264: 137 pgs: 137 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 375 op/s 2024-03-07T03:14:45.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:44 smithi148 bash[21362]: cluster 2024-03-07T03:14:44.321315+0000 mon.a (mon.0) 965 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2024-03-07T03:14:45.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:44 smithi148 bash[31149]: cluster 2024-03-07T03:14:43.109802+0000 mgr.y (mgr.24287) 227 : cluster 0 pgmap v264: 137 pgs: 137 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 375 op/s 2024-03-07T03:14:45.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:44 smithi148 bash[31149]: cluster 2024-03-07T03:14:44.321315+0000 mon.a (mon.0) 965 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2024-03-07T03:14:45.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:44 smithi195 bash[23221]: cluster 2024-03-07T03:14:43.109802+0000 mgr.y (mgr.24287) 227 : cluster 0 pgmap v264: 137 pgs: 137 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 375 op/s 2024-03-07T03:14:45.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:44 smithi195 bash[23221]: cluster 2024-03-07T03:14:44.321315+0000 mon.a (mon.0) 965 : cluster 0 osdmap e146: 8 total, 8 up, 8 in 2024-03-07T03:14:46.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:45 smithi148 bash[21362]: audit 2024-03-07T03:14:45.282032+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.148:0/1155721604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:46.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:45 smithi148 bash[21362]: cluster 2024-03-07T03:14:45.319478+0000 mon.a (mon.0) 966 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2024-03-07T03:14:46.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:45 smithi148 bash[31149]: audit 2024-03-07T03:14:45.282032+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.148:0/1155721604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:46.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:45 smithi148 bash[31149]: cluster 2024-03-07T03:14:45.319478+0000 mon.a (mon.0) 966 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2024-03-07T03:14:46.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:45 smithi195 bash[23221]: audit 2024-03-07T03:14:45.282032+0000 mon.c (mon.2) 43 : audit 0 from='client.? 172.21.15.148:0/1155721604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:46.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:45 smithi195 bash[23221]: cluster 2024-03-07T03:14:45.319478+0000 mon.a (mon.0) 966 : cluster 0 osdmap e147: 8 total, 8 up, 8 in 2024-03-07T03:14:46.341 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:46.397 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:14:46.419 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:47.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:46 smithi148 bash[21362]: cluster 2024-03-07T03:14:45.111462+0000 mgr.y (mgr.24287) 228 : cluster 0 pgmap v267: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-03-07T03:14:47.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:46 smithi148 bash[21362]: cluster 2024-03-07T03:14:46.326547+0000 mon.a (mon.0) 967 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2024-03-07T03:14:47.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:46 smithi148 bash[31149]: cluster 2024-03-07T03:14:45.111462+0000 mgr.y (mgr.24287) 228 : cluster 0 pgmap v267: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-03-07T03:14:47.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:46 smithi148 bash[31149]: cluster 2024-03-07T03:14:46.326547+0000 mon.a (mon.0) 967 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2024-03-07T03:14:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:46 smithi195 bash[23221]: cluster 2024-03-07T03:14:45.111462+0000 mgr.y (mgr.24287) 228 : cluster 0 pgmap v267: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-03-07T03:14:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:46 smithi195 bash[23221]: cluster 2024-03-07T03:14:46.326547+0000 mon.a (mon.0) 967 : cluster 0 osdmap e148: 8 total, 8 up, 8 in 2024-03-07T03:14:48.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:48 smithi195 bash[23221]: cluster 2024-03-07T03:14:47.112284+0000 mgr.y (mgr.24287) 229 : cluster 0 pgmap v270: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-03-07T03:14:48.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:48 smithi195 bash[23221]: cluster 2024-03-07T03:14:47.330166+0000 mon.a (mon.0) 968 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2024-03-07T03:14:48.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:48 smithi195 bash[23221]: audit 2024-03-07T03:14:47.734769+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.148:0/50427323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:48.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:48 smithi195 bash[23221]: cluster 2024-03-07T03:14:48.310283+0000 mon.a (mon.0) 969 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[21362]: cluster 2024-03-07T03:14:47.112284+0000 mgr.y (mgr.24287) 229 : cluster 0 pgmap v270: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[21362]: cluster 2024-03-07T03:14:47.330166+0000 mon.a (mon.0) 968 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[21362]: audit 2024-03-07T03:14:47.734769+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.148:0/50427323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[21362]: cluster 2024-03-07T03:14:48.310283+0000 mon.a (mon.0) 969 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[31149]: cluster 2024-03-07T03:14:47.112284+0000 mgr.y (mgr.24287) 229 : cluster 0 pgmap v270: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 183 op/s 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[31149]: cluster 2024-03-07T03:14:47.330166+0000 mon.a (mon.0) 968 : cluster 0 osdmap e149: 8 total, 8 up, 8 in 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[31149]: audit 2024-03-07T03:14:47.734769+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.148:0/50427323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:48.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:48 smithi148 bash[31149]: cluster 2024-03-07T03:14:48.310283+0000 mon.a (mon.0) 969 : cluster 0 osdmap e150: 8 total, 8 up, 8 in 2024-03-07T03:14:49.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:49 smithi195 bash[23221]: audit 2024-03-07T03:14:49.051114+0000 mon.b (mon.1) 132 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:49.787 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:49 smithi148 bash[21362]: audit 2024-03-07T03:14:49.051114+0000 mon.b (mon.1) 132 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:49.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:49 smithi148 bash[31149]: audit 2024-03-07T03:14:49.051114+0000 mon.b (mon.1) 132 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:14:50.099 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:14:50.301 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:14:50.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: cluster 2024-03-07T03:14:49.112946+0000 mgr.y (mgr.24287) 230 : cluster 0 pgmap v273: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:50.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: cluster 2024-03-07T03:14:49.342528+0000 mon.a (mon.0) 970 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2024-03-07T03:14:50.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: cephadm 2024-03-07T03:14:49.714974+0000 mgr.y (mgr.24287) 231 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:14:50.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: audit 2024-03-07T03:14:49.725213+0000 mon.b (mon.1) 133 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:50.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: audit 2024-03-07T03:14:49.726479+0000 mon.a (mon.0) 971 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: audit 2024-03-07T03:14:49.728867+0000 mon.b (mon.1) 134 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: audit 2024-03-07T03:14:49.729629+0000 mon.a (mon.0) 972 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: audit 2024-03-07T03:14:49.731111+0000 mon.b (mon.1) 135 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: cephadm 2024-03-07T03:14:49.736977+0000 mgr.y (mgr.24287) 232 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[21362]: audit 2024-03-07T03:14:50.257012+0000 mon.a (mon.0) 973 : audit 0 from='client.? 172.21.15.148:0/1144767995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: cluster 2024-03-07T03:14:49.112946+0000 mgr.y (mgr.24287) 230 : cluster 0 pgmap v273: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: cluster 2024-03-07T03:14:49.342528+0000 mon.a (mon.0) 970 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: cephadm 2024-03-07T03:14:49.714974+0000 mgr.y (mgr.24287) 231 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: audit 2024-03-07T03:14:49.725213+0000 mon.b (mon.1) 133 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: audit 2024-03-07T03:14:49.726479+0000 mon.a (mon.0) 971 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:50.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: audit 2024-03-07T03:14:49.728867+0000 mon.b (mon.1) 134 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:14:50.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: audit 2024-03-07T03:14:49.729629+0000 mon.a (mon.0) 972 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:50.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: audit 2024-03-07T03:14:49.731111+0000 mon.b (mon.1) 135 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:50.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: cephadm 2024-03-07T03:14:49.736977+0000 mgr.y (mgr.24287) 232 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:14:50.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:50 smithi148 bash[31149]: audit 2024-03-07T03:14:50.257012+0000 mon.a (mon.0) 973 : audit 0 from='client.? 172.21.15.148:0/1144767995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:50.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: cluster 2024-03-07T03:14:49.112946+0000 mgr.y (mgr.24287) 230 : cluster 0 pgmap v273: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: cluster 2024-03-07T03:14:49.342528+0000 mon.a (mon.0) 970 : cluster 0 osdmap e151: 8 total, 8 up, 8 in 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: cephadm 2024-03-07T03:14:49.714974+0000 mgr.y (mgr.24287) 231 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: audit 2024-03-07T03:14:49.725213+0000 mon.b (mon.1) 133 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: audit 2024-03-07T03:14:49.726479+0000 mon.a (mon.0) 971 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: audit 2024-03-07T03:14:49.728867+0000 mon.b (mon.1) 134 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: audit 2024-03-07T03:14:49.729629+0000 mon.a (mon.0) 972 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: audit 2024-03-07T03:14:49.731111+0000 mon.b (mon.1) 135 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: cephadm 2024-03-07T03:14:49.736977+0000 mgr.y (mgr.24287) 232 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:14:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:50 smithi195 bash[23221]: audit 2024-03-07T03:14:50.257012+0000 mon.a (mon.0) 973 : audit 0 from='client.? 172.21.15.148:0/1144767995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:52.413 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.413 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.413 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.414 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.414 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.414 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.414 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.415 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.415 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:52 smithi195 bash[23221]: cluster 2024-03-07T03:14:51.114575+0000 mgr.y (mgr.24287) 233 : cluster 0 pgmap v275: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-03-07T03:14:52.415 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:52 smithi195 bash[23221]: cluster 2024-03-07T03:14:51.353338+0000 mon.a (mon.0) 974 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2024-03-07T03:14:52.415 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:52.786 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:52 smithi195 systemd[1]: Stopping Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:14:52.787 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:52 smithi148 bash[21362]: cluster 2024-03-07T03:14:51.114575+0000 mgr.y (mgr.24287) 233 : cluster 0 pgmap v275: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-03-07T03:14:52.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:52 smithi148 bash[21362]: cluster 2024-03-07T03:14:51.353338+0000 mon.a (mon.0) 974 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2024-03-07T03:14:52.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:52 smithi148 bash[31149]: cluster 2024-03-07T03:14:51.114575+0000 mgr.y (mgr.24287) 233 : cluster 0 pgmap v275: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 187 op/s 2024-03-07T03:14:52.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:52 smithi148 bash[31149]: cluster 2024-03-07T03:14:51.353338+0000 mon.a (mon.0) 974 : cluster 0 osdmap e152: 8 total, 8 up, 8 in 2024-03-07T03:14:53.549 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:53 smithi195 bash[23221]: cluster 2024-03-07T03:14:52.361951+0000 mon.a (mon.0) 975 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2024-03-07T03:14:53.549 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:53 smithi195 bash[23221]: audit 2024-03-07T03:14:52.703547+0000 mon.a (mon.0) 976 : audit 0 from='client.? 172.21.15.148:0/2986527430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:53.549 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:53 smithi195 bash[51439]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:14:53.549 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:53 smithi195 bash[51502]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:14:53.549 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:14:53.549 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-07T03:14:53.549 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: Stopped Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:53.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:53 smithi148 bash[21362]: cluster 2024-03-07T03:14:52.361951+0000 mon.a (mon.0) 975 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2024-03-07T03:14:53.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:53 smithi148 bash[21362]: audit 2024-03-07T03:14:52.703547+0000 mon.a (mon.0) 976 : audit 0 from='client.? 172.21.15.148:0/2986527430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:53.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:53 smithi148 bash[31149]: cluster 2024-03-07T03:14:52.361951+0000 mon.a (mon.0) 975 : cluster 0 osdmap e153: 8 total, 8 up, 8 in 2024-03-07T03:14:53.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:53 smithi148 bash[31149]: audit 2024-03-07T03:14:52.703547+0000 mon.a (mon.0) 976 : audit 0 from='client.? 172.21.15.148:0/2986527430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:53.788 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:14:53 smithi148 bash[58229]: ts=2024-03-07T03:14:53.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:14:53.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.813 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.813 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.814 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.814 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.814 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.814 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.815 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: Started Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:14:53.815 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:14:53 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:14:53.872 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsDataPool (12642 ms) 2024-03-07T03:14:53.872 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-03-07T03:14:53.915 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:53.930 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:14:54.001 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:14:54.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[31149]: cluster 2024-03-07T03:14:53.115410+0000 mgr.y (mgr.24287) 234 : cluster 0 pgmap v278: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 186 op/s 2024-03-07T03:14:54.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[31149]: cluster 2024-03-07T03:14:53.373252+0000 mon.a (mon.0) 977 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2024-03-07T03:14:54.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[31149]: audit 2024-03-07T03:14:53.741899+0000 mon.a (mon.0) 978 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:54.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[31149]: audit 2024-03-07T03:14:53.762613+0000 mon.a (mon.0) 979 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:54.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[31149]: audit 2024-03-07T03:14:53.807344+0000 mon.b (mon.1) 136 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:54.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[21362]: cluster 2024-03-07T03:14:53.115410+0000 mgr.y (mgr.24287) 234 : cluster 0 pgmap v278: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 186 op/s 2024-03-07T03:14:54.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[21362]: cluster 2024-03-07T03:14:53.373252+0000 mon.a (mon.0) 977 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2024-03-07T03:14:54.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[21362]: audit 2024-03-07T03:14:53.741899+0000 mon.a (mon.0) 978 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:54.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[21362]: audit 2024-03-07T03:14:53.762613+0000 mon.a (mon.0) 979 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:54.631 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[21362]: audit 2024-03-07T03:14:53.807344+0000 mon.b (mon.1) 136 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:54 smithi195 bash[23221]: cluster 2024-03-07T03:14:53.115410+0000 mgr.y (mgr.24287) 234 : cluster 0 pgmap v278: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 27 MiB/s rd, 9.8 KiB/s wr, 186 op/s 2024-03-07T03:14:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:54 smithi195 bash[23221]: cluster 2024-03-07T03:14:53.373252+0000 mon.a (mon.0) 977 : cluster 0 osdmap e154: 8 total, 8 up, 8 in 2024-03-07T03:14:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:54 smithi195 bash[23221]: audit 2024-03-07T03:14:53.741899+0000 mon.a (mon.0) 978 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:54 smithi195 bash[23221]: audit 2024-03-07T03:14:53.762613+0000 mon.a (mon.0) 979 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:14:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:54 smithi195 bash[23221]: audit 2024-03-07T03:14:53.807344+0000 mon.b (mon.1) 136 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:14:55.037 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:14:54 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:14:54] "GET /metrics HTTP/1.1" 200 38771 "" "Prometheus/2.43.0" 2024-03-07T03:14:55.109 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:54 smithi195 bash[51559]: debug 2024-03-07T03:14:54.965+0000 7fd59858f700 1 -- 172.21.15.195:0/1696790864 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x558ebd9c25a0 con 0x558ebe746800 2024-03-07T03:14:55.563 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:55 smithi195 bash[51559]: debug 2024-03-07T03:14:55.117+0000 7fd5a2791200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:14:55.780 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:14:55.809 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:14:55.810 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:14:55.810 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:14:56.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:55 smithi148 bash[31149]: cluster 2024-03-07T03:14:54.764444+0000 mon.a (mon.0) 980 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2024-03-07T03:14:56.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:55 smithi148 bash[31149]: audit 2024-03-07T03:14:55.150899+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.148:0/1059067700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:56.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:55 smithi148 bash[21362]: cluster 2024-03-07T03:14:54.764444+0000 mon.a (mon.0) 980 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2024-03-07T03:14:56.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:55 smithi148 bash[21362]: audit 2024-03-07T03:14:55.150899+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.148:0/1059067700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:56.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:55 smithi195 bash[23221]: cluster 2024-03-07T03:14:54.764444+0000 mon.a (mon.0) 980 : cluster 0 osdmap e155: 8 total, 8 up, 8 in 2024-03-07T03:14:56.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:55 smithi195 bash[23221]: audit 2024-03-07T03:14:55.150899+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.148:0/1059067700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:57.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:56 smithi148 bash[21362]: cluster 2024-03-07T03:14:55.117029+0000 mgr.y (mgr.24287) 235 : cluster 0 pgmap v281: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-03-07T03:14:57.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:56 smithi148 bash[21362]: cluster 2024-03-07T03:14:55.774159+0000 mon.a (mon.0) 982 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2024-03-07T03:14:57.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:56 smithi148 bash[31149]: cluster 2024-03-07T03:14:55.117029+0000 mgr.y (mgr.24287) 235 : cluster 0 pgmap v281: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-03-07T03:14:57.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:56 smithi148 bash[31149]: cluster 2024-03-07T03:14:55.774159+0000 mon.a (mon.0) 982 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2024-03-07T03:14:57.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:56 smithi195 bash[23221]: cluster 2024-03-07T03:14:55.117029+0000 mgr.y (mgr.24287) 235 : cluster 0 pgmap v281: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-03-07T03:14:57.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:56 smithi195 bash[23221]: cluster 2024-03-07T03:14:55.774159+0000 mon.a (mon.0) 982 : cluster 0 osdmap e156: 8 total, 8 up, 8 in 2024-03-07T03:14:57.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:56 smithi195 bash[51559]: debug 2024-03-07T03:14:56.721+0000 7fd5a2791200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:14:57.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:56 smithi195 bash[51559]: debug 2024-03-07T03:14:56.829+0000 7fd5a2791200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:14:57.563 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:57 smithi195 bash[51559]: debug 2024-03-07T03:14:57.073+0000 7fd5a2791200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:14:57.798 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:14:57.817 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:14:57.817 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:14:57.836 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:14:57.883 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:14:58.037 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:57 smithi195 bash[23221]: cluster 2024-03-07T03:14:56.782254+0000 mon.a (mon.0) 983 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2024-03-07T03:14:58.037 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:57 smithi195 bash[23221]: audit 2024-03-07T03:14:57.587709+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.148:0/2493327866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:58.037 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:57 smithi195 bash[51559]: debug 2024-03-07T03:14:57.921+0000 7fd5a2791200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:14:58.037 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[51559]: debug 2024-03-07T03:14:58.045+0000 7fd5a2791200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:14:58.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:57 smithi148 bash[21362]: cluster 2024-03-07T03:14:56.782254+0000 mon.a (mon.0) 983 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2024-03-07T03:14:58.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:57 smithi148 bash[21362]: audit 2024-03-07T03:14:57.587709+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.148:0/2493327866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:58.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:57 smithi148 bash[31149]: cluster 2024-03-07T03:14:56.782254+0000 mon.a (mon.0) 983 : cluster 0 osdmap e157: 8 total, 8 up, 8 in 2024-03-07T03:14:58.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:57 smithi148 bash[31149]: audit 2024-03-07T03:14:57.587709+0000 mon.c (mon.2) 45 : audit 0 from='client.? 172.21.15.148:0/2493327866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:14:58.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[51559]: debug 2024-03-07T03:14:58.157+0000 7fd5a2791200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:14:58.769 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[51559]: debug 2024-03-07T03:14:58.525+0000 7fd5a2791200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:14:58.769 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[51559]: debug 2024-03-07T03:14:58.637+0000 7fd5a2791200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:14:58.769 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[51559]: debug 2024-03-07T03:14:58.753+0000 7fd5a2791200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:14:59.037 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:58 smithi148 bash[31149]: cluster 2024-03-07T03:14:57.117748+0000 mgr.y (mgr.24287) 236 : cluster 0 pgmap v284: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-03-07T03:14:59.037 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:58 smithi148 bash[31149]: cluster 2024-03-07T03:14:57.792078+0000 mon.a (mon.0) 984 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2024-03-07T03:14:59.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:14:58 smithi148 bash[31149]: cluster 2024-03-07T03:14:58.312223+0000 mon.a (mon.0) 985 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2024-03-07T03:14:59.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:58 smithi148 bash[21362]: cluster 2024-03-07T03:14:57.117748+0000 mgr.y (mgr.24287) 236 : cluster 0 pgmap v284: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-03-07T03:14:59.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:58 smithi148 bash[21362]: cluster 2024-03-07T03:14:57.792078+0000 mon.a (mon.0) 984 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2024-03-07T03:14:59.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:14:58 smithi148 bash[21362]: cluster 2024-03-07T03:14:58.312223+0000 mon.a (mon.0) 985 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2024-03-07T03:14:59.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[23221]: cluster 2024-03-07T03:14:57.117748+0000 mgr.y (mgr.24287) 236 : cluster 0 pgmap v284: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 38 MiB/s rd, 10 MiB/s wr, 378 op/s 2024-03-07T03:14:59.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[23221]: cluster 2024-03-07T03:14:57.792078+0000 mon.a (mon.0) 984 : cluster 0 osdmap e158: 8 total, 8 up, 8 in 2024-03-07T03:14:59.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:14:58 smithi195 bash[23221]: cluster 2024-03-07T03:14:58.312223+0000 mon.a (mon.0) 985 : cluster 0 osdmap e159: 8 total, 8 up, 8 in 2024-03-07T03:14:59.322 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:14:59.358 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:14:59.368 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:14:59.427 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:59 smithi195 bash[51559]: debug 2024-03-07T03:14:59.141+0000 7fd5a2791200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:14:59.709 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:59 smithi195 bash[51559]: debug 2024-03-07T03:14:59.433+0000 7fd5a2791200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:15:00.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:59 smithi195 bash[51559]: debug 2024-03-07T03:14:59.717+0000 7fd5a2791200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:15:00.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:14:59 smithi195 bash[51559]: debug 2024-03-07T03:14:59.829+0000 7fd5a2791200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:15:00.364 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[51559]: debug 2024-03-07T03:15:00.121+0000 7fd5a2791200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:15:00.365 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[23221]: cluster 2024-03-07T03:14:59.118185+0000 mgr.y (mgr.24287) 237 : cluster 0 pgmap v287: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:00.365 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[23221]: audit 2024-03-07T03:14:59.304415+0000 mon.a (mon.0) 986 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.365 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[23221]: audit 2024-03-07T03:14:59.313570+0000 mon.a (mon.0) 987 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.365 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[23221]: cluster 2024-03-07T03:14:59.319524+0000 mon.a (mon.0) 988 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2024-03-07T03:15:00.365 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[23221]: audit 2024-03-07T03:14:59.815133+0000 mon.a (mon.0) 989 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.365 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[23221]: audit 2024-03-07T03:14:59.830235+0000 mon.a (mon.0) 990 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.365 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[23221]: audit 2024-03-07T03:15:00.025010+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.148:0/894392623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:00.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[21362]: cluster 2024-03-07T03:14:59.118185+0000 mgr.y (mgr.24287) 237 : cluster 0 pgmap v287: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:00.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[21362]: audit 2024-03-07T03:14:59.304415+0000 mon.a (mon.0) 986 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[21362]: audit 2024-03-07T03:14:59.313570+0000 mon.a (mon.0) 987 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[21362]: cluster 2024-03-07T03:14:59.319524+0000 mon.a (mon.0) 988 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2024-03-07T03:15:00.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[21362]: audit 2024-03-07T03:14:59.815133+0000 mon.a (mon.0) 989 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[21362]: audit 2024-03-07T03:14:59.830235+0000 mon.a (mon.0) 990 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[21362]: audit 2024-03-07T03:15:00.025010+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.148:0/894392623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[31149]: cluster 2024-03-07T03:14:59.118185+0000 mgr.y (mgr.24287) 237 : cluster 0 pgmap v287: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[31149]: audit 2024-03-07T03:14:59.304415+0000 mon.a (mon.0) 986 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[31149]: audit 2024-03-07T03:14:59.313570+0000 mon.a (mon.0) 987 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[31149]: cluster 2024-03-07T03:14:59.319524+0000 mon.a (mon.0) 988 : cluster 0 osdmap e160: 8 total, 8 up, 8 in 2024-03-07T03:15:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[31149]: audit 2024-03-07T03:14:59.815133+0000 mon.a (mon.0) 989 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[31149]: audit 2024-03-07T03:14:59.830235+0000 mon.a (mon.0) 990 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:00 smithi148 bash[31149]: audit 2024-03-07T03:15:00.025010+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.148:0/894392623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:00.795 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[51559]: debug 2024-03-07T03:15:00.373+0000 7fd5a2791200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:01.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[51559]: debug 2024-03-07T03:15:00.801+0000 7fd5a2791200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:15:01.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:00 smithi195 bash[51559]: debug 2024-03-07T03:15:00.929+0000 7fd5a2791200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:15:01.563 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:01 smithi195 bash[51559]: debug 2024-03-07T03:15:01.177+0000 7fd5a2791200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:01.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:01 smithi195 bash[23221]: cluster 2024-03-07T03:15:00.332108+0000 mon.a (mon.0) 991 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2024-03-07T03:15:01.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:01 smithi195 bash[23221]: audit 2024-03-07T03:15:00.687605+0000 mon.a (mon.0) 992 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:01.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:01 smithi195 bash[23221]: audit 2024-03-07T03:15:00.704098+0000 mon.a (mon.0) 993 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:01.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:01 smithi148 bash[31149]: cluster 2024-03-07T03:15:00.332108+0000 mon.a (mon.0) 991 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2024-03-07T03:15:01.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:01 smithi148 bash[31149]: audit 2024-03-07T03:15:00.687605+0000 mon.a (mon.0) 992 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:01.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:01 smithi148 bash[31149]: audit 2024-03-07T03:15:00.704098+0000 mon.a (mon.0) 993 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:01.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:01 smithi148 bash[21362]: cluster 2024-03-07T03:15:00.332108+0000 mon.a (mon.0) 991 : cluster 0 osdmap e161: 8 total, 8 up, 8 in 2024-03-07T03:15:01.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:01 smithi148 bash[21362]: audit 2024-03-07T03:15:00.687605+0000 mon.a (mon.0) 992 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:01.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:01 smithi148 bash[21362]: audit 2024-03-07T03:15:00.704098+0000 mon.a (mon.0) 993 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:02 smithi195 bash[23221]: cluster 2024-03-07T03:15:01.118885+0000 mgr.y (mgr.24287) 238 : cluster 0 pgmap v290: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:02 smithi195 bash[23221]: cluster 2024-03-07T03:15:01.333014+0000 mon.a (mon.0) 994 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2024-03-07T03:15:02.564 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:02 smithi195 bash[51559]: debug 2024-03-07T03:15:02.209+0000 7fd5a2791200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:15:02.564 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:02 smithi195 bash[51559]: debug 2024-03-07T03:15:02.325+0000 7fd5a2791200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:15:02.787 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:02 smithi148 bash[21362]: cluster 2024-03-07T03:15:01.118885+0000 mgr.y (mgr.24287) 238 : cluster 0 pgmap v290: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:02.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:02 smithi148 bash[21362]: cluster 2024-03-07T03:15:01.333014+0000 mon.a (mon.0) 994 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2024-03-07T03:15:02.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:02 smithi148 bash[31149]: cluster 2024-03-07T03:15:01.118885+0000 mgr.y (mgr.24287) 238 : cluster 0 pgmap v290: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:02.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:02 smithi148 bash[31149]: cluster 2024-03-07T03:15:01.333014+0000 mon.a (mon.0) 994 : cluster 0 osdmap e162: 8 total, 8 up, 8 in 2024-03-07T03:15:03.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:02 smithi195 bash[51559]: debug 2024-03-07T03:15:02.649+0000 7fd5a2791200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:15:03.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:02 smithi195 bash[51559]: debug 2024-03-07T03:15:02.769+0000 7fd5a2791200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:15:03.104 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:15:03.161 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:15:03.416 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:15:03.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[23221]: cluster 2024-03-07T03:15:02.351852+0000 mon.a (mon.0) 995 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2024-03-07T03:15:03.534 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[23221]: audit 2024-03-07T03:15:02.471636+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.148:0/698608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:03.534 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[51559]: debug 2024-03-07T03:15:03.249+0000 7fd5a2791200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:15:03.535 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[51559]: debug 2024-03-07T03:15:03.409+0000 7fd5a2791200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:15:03.535 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[51559]: [07/Mar/2024:03:15:03] ENGINE Bus STARTING 2024-03-07T03:15:03.787 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:15:03 smithi148 bash[58229]: ts=2024-03-07T03:15:03.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:15:03.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:03 smithi148 bash[21362]: cluster 2024-03-07T03:15:02.351852+0000 mon.a (mon.0) 995 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2024-03-07T03:15:03.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:03 smithi148 bash[21362]: audit 2024-03-07T03:15:02.471636+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.148:0/698608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:03.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:03 smithi148 bash[31149]: cluster 2024-03-07T03:15:02.351852+0000 mon.a (mon.0) 995 : cluster 0 osdmap e163: 8 total, 8 up, 8 in 2024-03-07T03:15:03.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:03 smithi148 bash[31149]: audit 2024-03-07T03:15:02.471636+0000 mon.c (mon.2) 47 : audit 0 from='client.? 172.21.15.148:0/698608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:03.813 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[51559]: CherryPy Checker: 2024-03-07T03:15:03.813 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[51559]: The Application mounted at '' has an empty config. 2024-03-07T03:15:03.813 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[51559]: [07/Mar/2024:03:15:03] ENGINE Serving on http://:::9283 2024-03-07T03:15:03.813 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:03 smithi195 bash[51559]: [07/Mar/2024:03:15:03] ENGINE Bus STARTED 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: cluster 2024-03-07T03:15:03.119731+0000 mgr.y (mgr.24287) 239 : cluster 0 pgmap v293: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: audit 2024-03-07T03:15:03.441088+0000 mon.b (mon.1) 137 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: audit 2024-03-07T03:15:03.442376+0000 mon.b (mon.1) 138 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: cluster 2024-03-07T03:15:03.442609+0000 mon.a (mon.0) 996 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: cluster 2024-03-07T03:15:03.442819+0000 mon.a (mon.0) 997 : cluster 0 Standby manager daemon x started 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: audit 2024-03-07T03:15:03.444544+0000 mon.b (mon.1) 139 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: audit 2024-03-07T03:15:03.445977+0000 mon.b (mon.1) 140 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21362]: audit 2024-03-07T03:15:04.051578+0000 mon.b (mon.1) 141 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: cluster 2024-03-07T03:15:03.119731+0000 mgr.y (mgr.24287) 239 : cluster 0 pgmap v293: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:04.629 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: audit 2024-03-07T03:15:03.441088+0000 mon.b (mon.1) 137 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:15:04.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: audit 2024-03-07T03:15:03.442376+0000 mon.b (mon.1) 138 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:04.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: cluster 2024-03-07T03:15:03.442609+0000 mon.a (mon.0) 996 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:15:04.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: cluster 2024-03-07T03:15:03.442819+0000 mon.a (mon.0) 997 : cluster 0 Standby manager daemon x started 2024-03-07T03:15:04.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: audit 2024-03-07T03:15:03.444544+0000 mon.b (mon.1) 139 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:15:04.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: audit 2024-03-07T03:15:03.445977+0000 mon.b (mon.1) 140 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:04.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[31149]: audit 2024-03-07T03:15:04.051578+0000 mon.b (mon.1) 141 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:04.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: cluster 2024-03-07T03:15:03.119731+0000 mgr.y (mgr.24287) 239 : cluster 0 pgmap v293: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: audit 2024-03-07T03:15:03.441088+0000 mon.b (mon.1) 137 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:15:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: audit 2024-03-07T03:15:03.442376+0000 mon.b (mon.1) 138 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: cluster 2024-03-07T03:15:03.442609+0000 mon.a (mon.0) 996 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:15:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: cluster 2024-03-07T03:15:03.442819+0000 mon.a (mon.0) 997 : cluster 0 Standby manager daemon x started 2024-03-07T03:15:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: audit 2024-03-07T03:15:03.444544+0000 mon.b (mon.1) 139 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:15:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: audit 2024-03-07T03:15:03.445977+0000 mon.b (mon.1) 140 : audit 0 from='mgr.? 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:04 smithi195 bash[23221]: audit 2024-03-07T03:15:04.051578+0000 mon.b (mon.1) 141 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:05.037 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:04 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:15:04] "GET /metrics HTTP/1.1" 200 38774 "" "Prometheus/2.43.0" 2024-03-07T03:15:05.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:05 smithi148 bash[21362]: cluster 2024-03-07T03:15:04.383868+0000 mon.a (mon.0) 998 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2024-03-07T03:15:05.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:05 smithi148 bash[21362]: cluster 2024-03-07T03:15:04.411310+0000 mon.a (mon.0) 999 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-03-07T03:15:05.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:05 smithi148 bash[21362]: audit 2024-03-07T03:15:04.909157+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.148:0/2464548749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:05.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:05 smithi148 bash[31149]: cluster 2024-03-07T03:15:04.383868+0000 mon.a (mon.0) 998 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2024-03-07T03:15:05.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:05 smithi148 bash[31149]: cluster 2024-03-07T03:15:04.411310+0000 mon.a (mon.0) 999 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-03-07T03:15:05.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:05 smithi148 bash[31149]: audit 2024-03-07T03:15:04.909157+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.148:0/2464548749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:05.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:05 smithi195 bash[23221]: cluster 2024-03-07T03:15:04.383868+0000 mon.a (mon.0) 998 : cluster 0 osdmap e164: 8 total, 8 up, 8 in 2024-03-07T03:15:05.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:05 smithi195 bash[23221]: cluster 2024-03-07T03:15:04.411310+0000 mon.a (mon.0) 999 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-03-07T03:15:05.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:05 smithi195 bash[23221]: audit 2024-03-07T03:15:04.909157+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.148:0/2464548749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:06.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:06 smithi148 bash[31149]: cluster 2024-03-07T03:15:05.121390+0000 mgr.y (mgr.24287) 240 : cluster 0 pgmap v295: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 5.0 MiB/s wr, 349 op/s 2024-03-07T03:15:06.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:06 smithi148 bash[31149]: cluster 2024-03-07T03:15:05.413443+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2024-03-07T03:15:06.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:06 smithi148 bash[21362]: cluster 2024-03-07T03:15:05.121390+0000 mgr.y (mgr.24287) 240 : cluster 0 pgmap v295: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 5.0 MiB/s wr, 349 op/s 2024-03-07T03:15:06.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:06 smithi148 bash[21362]: cluster 2024-03-07T03:15:05.413443+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2024-03-07T03:15:06.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:06 smithi195 bash[23221]: cluster 2024-03-07T03:15:05.121390+0000 mgr.y (mgr.24287) 240 : cluster 0 pgmap v295: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 5.0 MiB/s wr, 349 op/s 2024-03-07T03:15:06.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:06 smithi195 bash[23221]: cluster 2024-03-07T03:15:05.413443+0000 mon.a (mon.0) 1000 : cluster 0 osdmap e165: 8 total, 8 up, 8 in 2024-03-07T03:15:06.914 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (13041 ms) 2024-03-07T03:15:06.914 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-03-07T03:15:06.988 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:07.011 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:15:07.082 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:15:07.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:07 smithi148 bash[21362]: cluster 2024-03-07T03:15:06.412447+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2024-03-07T03:15:07.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:07 smithi148 bash[21362]: audit 2024-03-07T03:15:07.369984+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.148:0/312393392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:07.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:07 smithi148 bash[31149]: cluster 2024-03-07T03:15:06.412447+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2024-03-07T03:15:07.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:07 smithi148 bash[31149]: audit 2024-03-07T03:15:07.369984+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.148:0/312393392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:07 smithi195 bash[23221]: cluster 2024-03-07T03:15:06.412447+0000 mon.a (mon.0) 1001 : cluster 0 osdmap e166: 8 total, 8 up, 8 in 2024-03-07T03:15:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:07 smithi195 bash[23221]: audit 2024-03-07T03:15:07.369984+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.148:0/312393392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:08.340 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:15:08.372 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:15:08.373 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:15:08.373 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:15:08.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:08 smithi148 bash[21362]: cluster 2024-03-07T03:15:07.121845+0000 mgr.y (mgr.24287) 241 : cluster 0 pgmap v298: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 5.0 MiB/s wr, 351 op/s 2024-03-07T03:15:08.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:08 smithi148 bash[21362]: cluster 2024-03-07T03:15:07.439395+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2024-03-07T03:15:08.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:08 smithi148 bash[21362]: cluster 2024-03-07T03:15:08.319219+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2024-03-07T03:15:08.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:08 smithi148 bash[31149]: cluster 2024-03-07T03:15:07.121845+0000 mgr.y (mgr.24287) 241 : cluster 0 pgmap v298: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 5.0 MiB/s wr, 351 op/s 2024-03-07T03:15:08.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:08 smithi148 bash[31149]: cluster 2024-03-07T03:15:07.439395+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2024-03-07T03:15:08.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:08 smithi148 bash[31149]: cluster 2024-03-07T03:15:08.319219+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2024-03-07T03:15:08.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:08 smithi195 bash[23221]: cluster 2024-03-07T03:15:07.121845+0000 mgr.y (mgr.24287) 241 : cluster 0 pgmap v298: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 5.0 MiB/s wr, 351 op/s 2024-03-07T03:15:08.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:08 smithi195 bash[23221]: cluster 2024-03-07T03:15:07.439395+0000 mon.a (mon.0) 1002 : cluster 0 osdmap e167: 8 total, 8 up, 8 in 2024-03-07T03:15:08.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:08 smithi195 bash[23221]: cluster 2024-03-07T03:15:08.319219+0000 mon.a (mon.0) 1003 : cluster 0 osdmap e168: 8 total, 8 up, 8 in 2024-03-07T03:15:10.468 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:15:10.486 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:15:10.487 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:15:10.508 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:15:10.571 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:15:10.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:10 smithi148 bash[21362]: cluster 2024-03-07T03:15:09.122540+0000 mgr.y (mgr.24287) 242 : cluster 0 pgmap v301: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:10.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:10 smithi148 bash[21362]: cluster 2024-03-07T03:15:09.452898+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2024-03-07T03:15:10.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:10 smithi148 bash[21362]: audit 2024-03-07T03:15:09.814711+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.148:0/4095543401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:10.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:10 smithi148 bash[31149]: cluster 2024-03-07T03:15:09.122540+0000 mgr.y (mgr.24287) 242 : cluster 0 pgmap v301: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:10.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:10 smithi148 bash[31149]: cluster 2024-03-07T03:15:09.452898+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2024-03-07T03:15:10.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:10 smithi148 bash[31149]: audit 2024-03-07T03:15:09.814711+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.148:0/4095543401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:10.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:10 smithi195 bash[23221]: cluster 2024-03-07T03:15:09.122540+0000 mgr.y (mgr.24287) 242 : cluster 0 pgmap v301: 137 pgs: 137 active+clean; 469 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:10.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:10 smithi195 bash[23221]: cluster 2024-03-07T03:15:09.452898+0000 mon.a (mon.0) 1004 : cluster 0 osdmap e169: 8 total, 8 up, 8 in 2024-03-07T03:15:10.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:10 smithi195 bash[23221]: audit 2024-03-07T03:15:09.814711+0000 mon.c (mon.2) 50 : audit 0 from='client.? 172.21.15.148:0/4095543401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:11.267 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21615]: debug 2024-03-07T03:15:11.018+0000 7f37da773700 -1 mgr handle_mgr_map I was active but no longer am 2024-03-07T03:15:11.267 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21615]: ignoring --setuser ceph since I am not root 2024-03-07T03:15:11.267 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21615]: ignoring --setgroup ceph since I am not root 2024-03-07T03:15:11.267 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21615]: debug 2024-03-07T03:15:11.230+0000 7f0d15860200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: [07/Mar/2024:03:15:11] ENGINE Bus STOPPING 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: [07/Mar/2024:03:15:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: [07/Mar/2024:03:15:11] ENGINE Bus STOPPED 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: [07/Mar/2024:03:15:11] ENGINE Bus STARTING 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd50340b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd50340b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd50340b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd50340b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd50340b700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd500405700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd500405700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd500405700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd500405700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: debug 2024-03-07T03:15:11.217+0000 7fd500405700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:15:11.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: [07/Mar/2024:03:15:11] ENGINE Serving on http://:::9283 2024-03-07T03:15:11.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[51559]: [07/Mar/2024:03:15:11] ENGINE Bus STARTED 2024-03-07T03:15:11.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: cluster 2024-03-07T03:15:10.459884+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:10.858487+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:10.876005+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:10.876917+0000 mon.b (mon.1) 142 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:10.879096+0000 mon.b (mon.1) 143 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:10.895428+0000 mon.a (mon.0) 1008 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:10.988658+0000 mon.b (mon.1) 144 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: cephadm 2024-03-07T03:15:10.998098+0000 mgr.y (mgr.24287) 243 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.003251+0000 mon.b (mon.1) 145 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: cephadm 2024-03-07T03:15:11.006191+0000 mgr.y (mgr.24287) 244 : cephadm 1 Failing over to other MGR 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.007488+0000 mon.a (mon.0) 1009 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: cluster 2024-03-07T03:15:11.021394+0000 mon.a (mon.0) 1010 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.022033+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: cluster 2024-03-07T03:15:11.022231+0000 mon.a (mon.0) 1012 : cluster 0 mgrmap e22: x(active, starting, since 0.0137963s) 2024-03-07T03:15:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.022544+0000 mon.b (mon.1) 146 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.023111+0000 mon.b (mon.1) 147 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.023644+0000 mon.b (mon.1) 148 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.024177+0000 mon.b (mon.1) 149 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.024921+0000 mon.b (mon.1) 150 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.025649+0000 mon.b (mon.1) 151 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.026048+0000 mon.b (mon.1) 152 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: cluster 2024-03-07T03:15:10.459884+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:10.858487+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:10.876005+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:10.876917+0000 mon.b (mon.1) 142 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:10.879096+0000 mon.b (mon.1) 143 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:10.895428+0000 mon.a (mon.0) 1008 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:10.988658+0000 mon.b (mon.1) 144 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: cephadm 2024-03-07T03:15:10.998098+0000 mgr.y (mgr.24287) 243 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.003251+0000 mon.b (mon.1) 145 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: cephadm 2024-03-07T03:15:11.006191+0000 mgr.y (mgr.24287) 244 : cephadm 1 Failing over to other MGR 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.007488+0000 mon.a (mon.0) 1009 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: cluster 2024-03-07T03:15:11.021394+0000 mon.a (mon.0) 1010 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.022033+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: cluster 2024-03-07T03:15:11.022231+0000 mon.a (mon.0) 1012 : cluster 0 mgrmap e22: x(active, starting, since 0.0137963s) 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.022544+0000 mon.b (mon.1) 146 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.023111+0000 mon.b (mon.1) 147 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.023644+0000 mon.b (mon.1) 148 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.024177+0000 mon.b (mon.1) 149 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:15:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.024921+0000 mon.b (mon.1) 150 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.025649+0000 mon.b (mon.1) 151 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.026048+0000 mon.b (mon.1) 152 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.026334+0000 mon.b (mon.1) 153 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.026912+0000 mon.b (mon.1) 154 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.027469+0000 mon.b (mon.1) 155 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.028301+0000 mon.b (mon.1) 156 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.028995+0000 mon.b (mon.1) 157 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.029396+0000 mon.b (mon.1) 158 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.029961+0000 mon.b (mon.1) 159 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:15:11.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.033135+0000 mon.b (mon.1) 160 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.033693+0000 mon.b (mon.1) 161 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.034647+0000 mon.b (mon.1) 162 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: cluster 2024-03-07T03:15:11.083709+0000 mon.a (mon.0) 1013 : cluster 1 Manager daemon x is now available 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.126309+0000 mon.b (mon.1) 163 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.128828+0000 mon.b (mon.1) 164 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.155132+0000 mon.b (mon.1) 165 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.182305+0000 mon.b (mon.1) 166 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.186486+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.200144+0000 mon.b (mon.1) 167 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[21362]: audit 2024-03-07T03:15:11.204448+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.026334+0000 mon.b (mon.1) 153 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:15:11.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.026912+0000 mon.b (mon.1) 154 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.027469+0000 mon.b (mon.1) 155 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.028301+0000 mon.b (mon.1) 156 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.028995+0000 mon.b (mon.1) 157 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.029396+0000 mon.b (mon.1) 158 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.029961+0000 mon.b (mon.1) 159 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.033135+0000 mon.b (mon.1) 160 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.033693+0000 mon.b (mon.1) 161 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.034647+0000 mon.b (mon.1) 162 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: cluster 2024-03-07T03:15:11.083709+0000 mon.a (mon.0) 1013 : cluster 1 Manager daemon x is now available 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.126309+0000 mon.b (mon.1) 163 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.128828+0000 mon.b (mon.1) 164 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.155132+0000 mon.b (mon.1) 165 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.182305+0000 mon.b (mon.1) 166 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:11.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.186486+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:11.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.200144+0000 mon.b (mon.1) 167 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:11.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:11 smithi148 bash[31149]: audit 2024-03-07T03:15:11.204448+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:11.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: cluster 2024-03-07T03:15:10.459884+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e170: 8 total, 8 up, 8 in 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:10.858487+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:10.876005+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:10.876917+0000 mon.b (mon.1) 142 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:10.879096+0000 mon.b (mon.1) 143 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:10.895428+0000 mon.a (mon.0) 1008 : audit 1 from='mgr.24287 ' entity='mgr.y' 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:10.988658+0000 mon.b (mon.1) 144 : audit 0 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: cephadm 2024-03-07T03:15:10.998098+0000 mgr.y (mgr.24287) 243 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.003251+0000 mon.b (mon.1) 145 : audit 1 from='mgr.24287 172.21.15.148:0/1570329394' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: cephadm 2024-03-07T03:15:11.006191+0000 mgr.y (mgr.24287) 244 : cephadm 1 Failing over to other MGR 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.007488+0000 mon.a (mon.0) 1009 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: cluster 2024-03-07T03:15:11.021394+0000 mon.a (mon.0) 1010 : cluster 0 osdmap e171: 8 total, 8 up, 8 in 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.022033+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.24287 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-07T03:15:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: cluster 2024-03-07T03:15:11.022231+0000 mon.a (mon.0) 1012 : cluster 0 mgrmap e22: x(active, starting, since 0.0137963s) 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.022544+0000 mon.b (mon.1) 146 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.023111+0000 mon.b (mon.1) 147 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.023644+0000 mon.b (mon.1) 148 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.024177+0000 mon.b (mon.1) 149 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.024921+0000 mon.b (mon.1) 150 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.025649+0000 mon.b (mon.1) 151 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.026048+0000 mon.b (mon.1) 152 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.026334+0000 mon.b (mon.1) 153 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.026912+0000 mon.b (mon.1) 154 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.027469+0000 mon.b (mon.1) 155 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.028301+0000 mon.b (mon.1) 156 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.028995+0000 mon.b (mon.1) 157 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.029396+0000 mon.b (mon.1) 158 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:15:11.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.029961+0000 mon.b (mon.1) 159 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.033135+0000 mon.b (mon.1) 160 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.033693+0000 mon.b (mon.1) 161 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.034647+0000 mon.b (mon.1) 162 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: cluster 2024-03-07T03:15:11.083709+0000 mon.a (mon.0) 1013 : cluster 1 Manager daemon x is now available 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.126309+0000 mon.b (mon.1) 163 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.128828+0000 mon.b (mon.1) 164 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.155132+0000 mon.b (mon.1) 165 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.182305+0000 mon.b (mon.1) 166 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.186486+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.200144+0000 mon.b (mon.1) 167 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:11.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:11 smithi195 bash[23221]: audit 2024-03-07T03:15:11.204448+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-07T03:15:12.027 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:12.063 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:15:12.080 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 0 2024-03-07T03:15:12.108 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cephadm 2024-03-07T03:15:11.585616+0000 mgr.x (mgr.24580) 1 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Bus STARTING 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cephadm 2024-03-07T03:15:11.613872+0000 mgr.x (mgr.24580) 2 : cephadm 1 Deploying cephadm binary to smithi195 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cephadm 2024-03-07T03:15:11.629646+0000 mgr.x (mgr.24580) 3 : cephadm 1 Deploying cephadm binary to smithi148 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cephadm 2024-03-07T03:15:11.709278+0000 mgr.x (mgr.24580) 4 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Serving on http://172.21.15.195:8765 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cephadm 2024-03-07T03:15:11.889739+0000 mgr.x (mgr.24580) 5 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Serving on https://172.21.15.195:7150 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cephadm 2024-03-07T03:15:11.889981+0000 mgr.x (mgr.24580) 6 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Bus STARTED 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cluster 2024-03-07T03:15:12.019830+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2024-03-07T03:15:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: cluster 2024-03-07T03:15:12.031530+0000 mon.a (mon.0) 1017 : cluster 0 mgrmap e23: x(active, since 1.02309s) 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21362]: audit 2024-03-07T03:15:12.309923+0000 mon.a (mon.0) 1018 : audit 0 from='client.? 172.21.15.148:0/44486135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cephadm 2024-03-07T03:15:11.585616+0000 mgr.x (mgr.24580) 1 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Bus STARTING 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cephadm 2024-03-07T03:15:11.613872+0000 mgr.x (mgr.24580) 2 : cephadm 1 Deploying cephadm binary to smithi195 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cephadm 2024-03-07T03:15:11.629646+0000 mgr.x (mgr.24580) 3 : cephadm 1 Deploying cephadm binary to smithi148 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cephadm 2024-03-07T03:15:11.709278+0000 mgr.x (mgr.24580) 4 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Serving on http://172.21.15.195:8765 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cephadm 2024-03-07T03:15:11.889739+0000 mgr.x (mgr.24580) 5 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Serving on https://172.21.15.195:7150 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cephadm 2024-03-07T03:15:11.889981+0000 mgr.x (mgr.24580) 6 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Bus STARTED 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cluster 2024-03-07T03:15:12.019830+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: cluster 2024-03-07T03:15:12.031530+0000 mon.a (mon.0) 1017 : cluster 0 mgrmap e23: x(active, since 1.02309s) 2024-03-07T03:15:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[31149]: audit 2024-03-07T03:15:12.309923+0000 mon.a (mon.0) 1018 : audit 0 from='client.? 172.21.15.148:0/44486135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cephadm 2024-03-07T03:15:11.585616+0000 mgr.x (mgr.24580) 1 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Bus STARTING 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cephadm 2024-03-07T03:15:11.613872+0000 mgr.x (mgr.24580) 2 : cephadm 1 Deploying cephadm binary to smithi195 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cephadm 2024-03-07T03:15:11.629646+0000 mgr.x (mgr.24580) 3 : cephadm 1 Deploying cephadm binary to smithi148 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cephadm 2024-03-07T03:15:11.709278+0000 mgr.x (mgr.24580) 4 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Serving on http://172.21.15.195:8765 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cephadm 2024-03-07T03:15:11.889739+0000 mgr.x (mgr.24580) 5 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Serving on https://172.21.15.195:7150 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cephadm 2024-03-07T03:15:11.889981+0000 mgr.x (mgr.24580) 6 : cephadm 1 [07/Mar/2024:03:15:11] ENGINE Bus STARTED 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cluster 2024-03-07T03:15:12.019830+0000 mon.a (mon.0) 1016 : cluster 0 osdmap e172: 8 total, 8 up, 8 in 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: cluster 2024-03-07T03:15:12.031530+0000 mon.a (mon.0) 1017 : cluster 0 mgrmap e23: x(active, since 1.02309s) 2024-03-07T03:15:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:12 smithi195 bash[23221]: audit 2024-03-07T03:15:12.309923+0000 mon.a (mon.0) 1018 : audit 0 from='client.? 172.21.15.148:0/44486135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:13.192 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21615]: debug 2024-03-07T03:15:12.850+0000 7f0d15860200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:15:13.192 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:12 smithi148 bash[21615]: debug 2024-03-07T03:15:12.946+0000 7f0d15860200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:15:13.444 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[21615]: debug 2024-03-07T03:15:13.202+0000 7f0d15860200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:15:13.788 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[58229]: ts=2024-03-07T03:15:13.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:15:13.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[31149]: cluster 2024-03-07T03:15:12.041438+0000 mgr.x (mgr.24580) 7 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:13.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[31149]: cluster 2024-03-07T03:15:13.042687+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2024-03-07T03:15:13.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[31149]: cluster 2024-03-07T03:15:13.316030+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2024-03-07T03:15:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[21362]: cluster 2024-03-07T03:15:12.041438+0000 mgr.x (mgr.24580) 7 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[21362]: cluster 2024-03-07T03:15:13.042687+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2024-03-07T03:15:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:13 smithi148 bash[21362]: cluster 2024-03-07T03:15:13.316030+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2024-03-07T03:15:13.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:13 smithi195 bash[23221]: cluster 2024-03-07T03:15:12.041438+0000 mgr.x (mgr.24580) 7 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:13.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:13 smithi195 bash[23221]: cluster 2024-03-07T03:15:13.042687+0000 mon.a (mon.0) 1019 : cluster 0 osdmap e173: 8 total, 8 up, 8 in 2024-03-07T03:15:13.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:13 smithi195 bash[23221]: cluster 2024-03-07T03:15:13.316030+0000 mon.a (mon.0) 1020 : cluster 0 osdmap e174: 8 total, 8 up, 8 in 2024-03-07T03:15:14.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21615]: debug 2024-03-07T03:15:14.010+0000 7f0d15860200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:15:14.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21615]: debug 2024-03-07T03:15:14.126+0000 7f0d15860200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:15:14.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21615]: debug 2024-03-07T03:15:14.230+0000 7f0d15860200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:15:14.755 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21615]: debug 2024-03-07T03:15:14.630+0000 7f0d15860200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:15:14.755 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21362]: cluster 2024-03-07T03:15:13.033576+0000 mgr.x (mgr.24580) 8 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:14.755 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21362]: cluster 2024-03-07T03:15:14.335764+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2024-03-07T03:15:14.756 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[31149]: cluster 2024-03-07T03:15:13.033576+0000 mgr.x (mgr.24580) 8 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:14.756 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[31149]: cluster 2024-03-07T03:15:14.335764+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2024-03-07T03:15:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:14 smithi195 bash[23221]: cluster 2024-03-07T03:15:13.033576+0000 mgr.x (mgr.24580) 8 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:14 smithi195 bash[23221]: cluster 2024-03-07T03:15:14.335764+0000 mon.a (mon.0) 1021 : cluster 0 osdmap e175: 8 total, 8 up, 8 in 2024-03-07T03:15:14.866 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:15:15.037 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21615]: debug 2024-03-07T03:15:14.766+0000 7f0d15860200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:15:15.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:14 smithi148 bash[21615]: debug 2024-03-07T03:15:14.890+0000 7f0d15860200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:15:15.065 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:15:15.562 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[21615]: debug 2024-03-07T03:15:15.310+0000 7f0d15860200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:15:15.563 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[21362]: cluster 2024-03-07T03:15:14.466646+0000 mon.a (mon.0) 1022 : cluster 0 mgrmap e24: x(active, since 3s) 2024-03-07T03:15:15.563 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[21362]: audit 2024-03-07T03:15:14.779684+0000 mon.a (mon.0) 1023 : audit 0 from='client.? 172.21.15.148:0/3413906493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:15.563 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[31149]: cluster 2024-03-07T03:15:14.466646+0000 mon.a (mon.0) 1022 : cluster 0 mgrmap e24: x(active, since 3s) 2024-03-07T03:15:15.563 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[31149]: audit 2024-03-07T03:15:14.779684+0000 mon.a (mon.0) 1023 : audit 0 from='client.? 172.21.15.148:0/3413906493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:15.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:15 smithi195 bash[23221]: cluster 2024-03-07T03:15:14.466646+0000 mon.a (mon.0) 1022 : cluster 0 mgrmap e24: x(active, since 3s) 2024-03-07T03:15:15.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:15 smithi195 bash[23221]: audit 2024-03-07T03:15:14.779684+0000 mon.a (mon.0) 1023 : audit 0 from='client.? 172.21.15.148:0/3413906493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:15.824 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[21615]: debug 2024-03-07T03:15:15.574+0000 7f0d15860200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:15:16.219 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[21615]: debug 2024-03-07T03:15:15.838+0000 7f0d15860200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:15:16.219 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:15 smithi148 bash[21615]: debug 2024-03-07T03:15:15.950+0000 7f0d15860200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:15:16.469 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:16 smithi148 bash[21362]: cluster 2024-03-07T03:15:15.034293+0000 mgr.x (mgr.24580) 9 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:16.469 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:16 smithi148 bash[21362]: cluster 2024-03-07T03:15:15.489865+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2024-03-07T03:15:16.470 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:16 smithi148 bash[21615]: debug 2024-03-07T03:15:16.230+0000 7f0d15860200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:15:16.470 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:16 smithi148 bash[21615]: debug 2024-03-07T03:15:16.474+0000 7f0d15860200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:16.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:16 smithi148 bash[31149]: cluster 2024-03-07T03:15:15.034293+0000 mgr.x (mgr.24580) 9 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:16.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:16 smithi148 bash[31149]: cluster 2024-03-07T03:15:15.489865+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2024-03-07T03:15:16.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:16 smithi195 bash[23221]: cluster 2024-03-07T03:15:15.034293+0000 mgr.x (mgr.24580) 9 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:16.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:16 smithi195 bash[23221]: cluster 2024-03-07T03:15:15.489865+0000 mon.a (mon.0) 1024 : cluster 0 osdmap e176: 8 total, 8 up, 8 in 2024-03-07T03:15:17.266 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:16 smithi148 bash[21615]: debug 2024-03-07T03:15:16.898+0000 7f0d15860200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:15:17.266 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:17 smithi148 bash[21615]: debug 2024-03-07T03:15:17.026+0000 7f0d15860200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:15:17.537 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:17 smithi148 bash[21362]: cluster 2024-03-07T03:15:16.496394+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2024-03-07T03:15:17.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:17 smithi148 bash[21362]: audit 2024-03-07T03:15:17.221014+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.148:0/1353853683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:17.538 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:17 smithi148 bash[31149]: cluster 2024-03-07T03:15:16.496394+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2024-03-07T03:15:17.538 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:17 smithi148 bash[31149]: audit 2024-03-07T03:15:17.221014+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.148:0/1353853683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:17.538 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:17 smithi148 bash[21615]: debug 2024-03-07T03:15:17.278+0000 7f0d15860200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:17.803 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (10889 ms) 2024-03-07T03:15:17.803 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-03-07T03:15:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:17 smithi195 bash[23221]: cluster 2024-03-07T03:15:16.496394+0000 mon.a (mon.0) 1025 : cluster 0 osdmap e177: 8 total, 8 up, 8 in 2024-03-07T03:15:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:17 smithi195 bash[23221]: audit 2024-03-07T03:15:17.221014+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.148:0/1353853683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:17.872 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:17.897 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:15:17.993 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:15:18.788 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[21615]: debug 2024-03-07T03:15:18.398+0000 7f0d15860200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:15:18.788 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[21615]: debug 2024-03-07T03:15:18.518+0000 7f0d15860200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:15:18.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[21362]: cluster 2024-03-07T03:15:17.035482+0000 mgr.x (mgr.24580) 10 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:18.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[21362]: cluster 2024-03-07T03:15:17.504959+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2024-03-07T03:15:18.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[31149]: cluster 2024-03-07T03:15:17.035482+0000 mgr.x (mgr.24580) 10 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:18.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[31149]: cluster 2024-03-07T03:15:17.504959+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2024-03-07T03:15:18.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:18 smithi195 bash[23221]: cluster 2024-03-07T03:15:17.035482+0000 mgr.x (mgr.24580) 10 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:15:18.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:18 smithi195 bash[23221]: cluster 2024-03-07T03:15:17.504959+0000 mon.a (mon.0) 1026 : cluster 0 osdmap e178: 8 total, 8 up, 8 in 2024-03-07T03:15:19.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[21615]: debug 2024-03-07T03:15:18.818+0000 7f0d15860200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:15:19.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:18 smithi148 bash[21615]: debug 2024-03-07T03:15:18.934+0000 7f0d15860200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:15:19.525 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:15:19.552 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:15:19.552 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:15:19.552 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:15:19.643 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21362]: cluster 2024-03-07T03:15:18.503521+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2024-03-07T03:15:19.644 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[31149]: cluster 2024-03-07T03:15:18.503521+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2024-03-07T03:15:19.644 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21615]: debug 2024-03-07T03:15:19.398+0000 7f0d15860200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:15:19.644 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21615]: debug 2024-03-07T03:15:19.522+0000 7f0d15860200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:15:19.644 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21615]: [07/Mar/2024:03:15:19] ENGINE Bus STARTING 2024-03-07T03:15:19.644 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21615]: CherryPy Checker: 2024-03-07T03:15:19.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:19 smithi195 bash[23221]: cluster 2024-03-07T03:15:18.503521+0000 mon.a (mon.0) 1027 : cluster 0 osdmap e179: 8 total, 8 up, 8 in 2024-03-07T03:15:20.037 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21615]: The Application mounted at '' has an empty config. 2024-03-07T03:15:20.037 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21615]: [07/Mar/2024:03:15:19] ENGINE Serving on http://:::9283 2024-03-07T03:15:20.037 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:19 smithi148 bash[21615]: [07/Mar/2024:03:15:19] ENGINE Bus STARTED 2024-03-07T03:15:20.524 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[58229]: ts=2024-03-07T03:15:20.152Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=6 2024-03-07T03:15:20.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: cluster 2024-03-07T03:15:19.036242+0000 mgr.x (mgr.24580) 11 : cluster 0 pgmap v15: 137 pgs: 137 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 541 op/s 2024-03-07T03:15:20.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: cluster 2024-03-07T03:15:19.524675+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2024-03-07T03:15:20.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: cluster 2024-03-07T03:15:19.553877+0000 mon.a (mon.0) 1029 : cluster 0 Standby manager daemon y started 2024-03-07T03:15:20.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: audit 2024-03-07T03:15:19.554276+0000 mon.a (mon.0) 1030 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: audit 2024-03-07T03:15:19.554913+0000 mon.a (mon.0) 1031 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: audit 2024-03-07T03:15:19.557318+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: audit 2024-03-07T03:15:19.558696+0000 mon.a (mon.0) 1033 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[21362]: audit 2024-03-07T03:15:19.669935+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.148:0/14741895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: cluster 2024-03-07T03:15:19.036242+0000 mgr.x (mgr.24580) 11 : cluster 0 pgmap v15: 137 pgs: 137 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 541 op/s 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: cluster 2024-03-07T03:15:19.524675+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: cluster 2024-03-07T03:15:19.553877+0000 mon.a (mon.0) 1029 : cluster 0 Standby manager daemon y started 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: audit 2024-03-07T03:15:19.554276+0000 mon.a (mon.0) 1030 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: audit 2024-03-07T03:15:19.554913+0000 mon.a (mon.0) 1031 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: audit 2024-03-07T03:15:19.557318+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: audit 2024-03-07T03:15:19.558696+0000 mon.a (mon.0) 1033 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:20.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:20 smithi148 bash[31149]: audit 2024-03-07T03:15:19.669935+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.148:0/14741895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:20.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: cluster 2024-03-07T03:15:19.036242+0000 mgr.x (mgr.24580) 11 : cluster 0 pgmap v15: 137 pgs: 137 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 541 op/s 2024-03-07T03:15:20.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: cluster 2024-03-07T03:15:19.524675+0000 mon.a (mon.0) 1028 : cluster 0 osdmap e180: 8 total, 8 up, 8 in 2024-03-07T03:15:20.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: cluster 2024-03-07T03:15:19.553877+0000 mon.a (mon.0) 1029 : cluster 0 Standby manager daemon y started 2024-03-07T03:15:20.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: audit 2024-03-07T03:15:19.554276+0000 mon.a (mon.0) 1030 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:15:20.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: audit 2024-03-07T03:15:19.554913+0000 mon.a (mon.0) 1031 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:15:20.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: audit 2024-03-07T03:15:19.557318+0000 mon.a (mon.0) 1032 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:15:20.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: audit 2024-03-07T03:15:19.558696+0000 mon.a (mon.0) 1033 : audit 0 from='mgr.? 172.21.15.148:0/201739820' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:15:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:20 smithi195 bash[23221]: audit 2024-03-07T03:15:19.669935+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.148:0/14741895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:21.572 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:15:21.587 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:15:21.587 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:15:21.612 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:15:21.664 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:15:21.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:21 smithi195 bash[23221]: cluster 2024-03-07T03:15:20.533233+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2024-03-07T03:15:21.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:21 smithi195 bash[23221]: cluster 2024-03-07T03:15:20.556265+0000 mon.a (mon.0) 1035 : cluster 0 mgrmap e25: x(active, since 9s), standbys: y 2024-03-07T03:15:21.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:21 smithi195 bash[23221]: audit 2024-03-07T03:15:20.556446+0000 mon.b (mon.1) 168 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:15:22.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:21 smithi148 bash[21362]: cluster 2024-03-07T03:15:20.533233+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2024-03-07T03:15:22.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:21 smithi148 bash[21362]: cluster 2024-03-07T03:15:20.556265+0000 mon.a (mon.0) 1035 : cluster 0 mgrmap e25: x(active, since 9s), standbys: y 2024-03-07T03:15:22.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:21 smithi148 bash[21362]: audit 2024-03-07T03:15:20.556446+0000 mon.b (mon.1) 168 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:15:22.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:21 smithi148 bash[31149]: cluster 2024-03-07T03:15:20.533233+0000 mon.a (mon.0) 1034 : cluster 0 osdmap e181: 8 total, 8 up, 8 in 2024-03-07T03:15:22.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:21 smithi148 bash[31149]: cluster 2024-03-07T03:15:20.556265+0000 mon.a (mon.0) 1035 : cluster 0 mgrmap e25: x(active, since 9s), standbys: y 2024-03-07T03:15:22.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:21 smithi148 bash[31149]: audit 2024-03-07T03:15:20.556446+0000 mon.b (mon.1) 168 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:15:22.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:22 smithi195 bash[23221]: cluster 2024-03-07T03:15:21.037160+0000 mgr.x (mgr.24580) 12 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 541 op/s 2024-03-07T03:15:22.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:22 smithi195 bash[23221]: cluster 2024-03-07T03:15:21.562334+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2024-03-07T03:15:22.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:22 smithi195 bash[23221]: audit 2024-03-07T03:15:22.110801+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.148:0/939179657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:23.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:22 smithi148 bash[21362]: cluster 2024-03-07T03:15:21.037160+0000 mgr.x (mgr.24580) 12 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 541 op/s 2024-03-07T03:15:23.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:22 smithi148 bash[21362]: cluster 2024-03-07T03:15:21.562334+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2024-03-07T03:15:23.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:22 smithi148 bash[21362]: audit 2024-03-07T03:15:22.110801+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.148:0/939179657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:23.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:22 smithi148 bash[31149]: cluster 2024-03-07T03:15:21.037160+0000 mgr.x (mgr.24580) 12 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 513 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 541 op/s 2024-03-07T03:15:23.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:22 smithi148 bash[31149]: cluster 2024-03-07T03:15:21.562334+0000 mon.a (mon.0) 1036 : cluster 0 osdmap e182: 8 total, 8 up, 8 in 2024-03-07T03:15:23.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:22 smithi148 bash[31149]: audit 2024-03-07T03:15:22.110801+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.148:0/939179657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:23.339 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:23.397 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:15:23.418 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:15:23.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:23 smithi195 bash[23221]: cluster 2024-03-07T03:15:22.557718+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2024-03-07T03:15:23.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:23 smithi195 bash[23221]: cluster 2024-03-07T03:15:23.330199+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2024-03-07T03:15:24.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:23 smithi148 bash[21362]: cluster 2024-03-07T03:15:22.557718+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2024-03-07T03:15:24.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:23 smithi148 bash[21362]: cluster 2024-03-07T03:15:23.330199+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2024-03-07T03:15:24.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:23 smithi148 bash[31149]: cluster 2024-03-07T03:15:22.557718+0000 mon.a (mon.0) 1037 : cluster 0 osdmap e183: 8 total, 8 up, 8 in 2024-03-07T03:15:24.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:23 smithi148 bash[31149]: cluster 2024-03-07T03:15:23.330199+0000 mon.a (mon.0) 1038 : cluster 0 osdmap e184: 8 total, 8 up, 8 in 2024-03-07T03:15:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:24 smithi195 bash[23221]: cluster 2024-03-07T03:15:23.037974+0000 mgr.x (mgr.24580) 13 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:15:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:24 smithi195 bash[23221]: audit 2024-03-07T03:15:24.542997+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.148:0/352930714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:25.037 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:24 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:15:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:15:25.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:24 smithi148 bash[21362]: cluster 2024-03-07T03:15:23.037974+0000 mgr.x (mgr.24580) 13 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:15:25.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:24 smithi148 bash[21362]: audit 2024-03-07T03:15:24.542997+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.148:0/352930714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:25.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:24 smithi148 bash[31149]: cluster 2024-03-07T03:15:23.037974+0000 mgr.x (mgr.24580) 13 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:15:25.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:24 smithi148 bash[31149]: audit 2024-03-07T03:15:24.542997+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.148:0/352930714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:26.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:25 smithi148 bash[21362]: cluster 2024-03-07T03:15:24.592393+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2024-03-07T03:15:26.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:25 smithi148 bash[21362]: audit 2024-03-07T03:15:25.198171+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:26.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:25 smithi148 bash[21362]: audit 2024-03-07T03:15:25.210359+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:26.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:25 smithi148 bash[31149]: cluster 2024-03-07T03:15:24.592393+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2024-03-07T03:15:26.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:25 smithi148 bash[31149]: audit 2024-03-07T03:15:25.198171+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:26.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:25 smithi148 bash[31149]: audit 2024-03-07T03:15:25.210359+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:26.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:25 smithi195 bash[23221]: cluster 2024-03-07T03:15:24.592393+0000 mon.a (mon.0) 1039 : cluster 0 osdmap e185: 8 total, 8 up, 8 in 2024-03-07T03:15:26.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:25 smithi195 bash[23221]: audit 2024-03-07T03:15:25.198171+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:26.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:25 smithi195 bash[23221]: audit 2024-03-07T03:15:25.210359+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: cluster 2024-03-07T03:15:25.038522+0000 mgr.x (mgr.24580) 14 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:15:27.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: cluster 2024-03-07T03:15:25.596137+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2024-03-07T03:15:27.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:25.649222+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:25.656734+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.155898+0000 mon.b (mon.1) 169 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:27.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.364528+0000 mon.a (mon.0) 1045 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.376864+0000 mon.b (mon.1) 170 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.377076+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.379004+0000 mon.b (mon.1) 171 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.380540+0000 mon.b (mon.1) 172 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.380549+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.382408+0000 mon.b (mon.1) 173 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.382572+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.384184+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.386056+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[31149]: audit 2024-03-07T03:15:26.395337+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: cluster 2024-03-07T03:15:25.038522+0000 mgr.x (mgr.24580) 14 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: cluster 2024-03-07T03:15:25.596137+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2024-03-07T03:15:27.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:25.649222+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:25.656734+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.155898+0000 mon.b (mon.1) 169 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.364528+0000 mon.a (mon.0) 1045 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.376864+0000 mon.b (mon.1) 170 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.377076+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.379004+0000 mon.b (mon.1) 171 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.380540+0000 mon.b (mon.1) 172 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.380549+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.382408+0000 mon.b (mon.1) 173 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.382572+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.384184+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.386056+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:26 smithi148 bash[21362]: audit 2024-03-07T03:15:26.395337+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: cluster 2024-03-07T03:15:25.038522+0000 mgr.x (mgr.24580) 14 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 12 MiB/s wr, 173 op/s 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: cluster 2024-03-07T03:15:25.596137+0000 mon.a (mon.0) 1042 : cluster 0 osdmap e186: 8 total, 8 up, 8 in 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:25.649222+0000 mon.a (mon.0) 1043 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:25.656734+0000 mon.a (mon.0) 1044 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.155898+0000 mon.b (mon.1) 169 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.364528+0000 mon.a (mon.0) 1045 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.376864+0000 mon.b (mon.1) 170 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.377076+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.379004+0000 mon.b (mon.1) 171 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.380540+0000 mon.b (mon.1) 172 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.380549+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.382408+0000 mon.b (mon.1) 173 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.382572+0000 mon.a (mon.0) 1048 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.384184+0000 mon.a (mon.0) 1049 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.386056+0000 mon.a (mon.0) 1050 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:27.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:26 smithi195 bash[23221]: audit 2024-03-07T03:15:26.395337+0000 mon.a (mon.0) 1051 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:27.338 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 0 2024-03-07T03:15:27.457 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:15:27.629 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:15:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[21362]: cephadm 2024-03-07T03:15:26.383911+0000 mgr.x (mgr.24580) 15 : cephadm 1 Adjusting osd_memory_target on smithi148 to 2524M 2024-03-07T03:15:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[21362]: cluster 2024-03-07T03:15:26.597382+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2024-03-07T03:15:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[21362]: audit 2024-03-07T03:15:26.822032+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[21362]: audit 2024-03-07T03:15:26.840519+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[21362]: audit 2024-03-07T03:15:27.006532+0000 mon.a (mon.0) 1055 : audit 0 from='client.? 172.21.15.148:0/1036522695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[31149]: cephadm 2024-03-07T03:15:26.383911+0000 mgr.x (mgr.24580) 15 : cephadm 1 Adjusting osd_memory_target on smithi148 to 2524M 2024-03-07T03:15:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[31149]: cluster 2024-03-07T03:15:26.597382+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2024-03-07T03:15:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[31149]: audit 2024-03-07T03:15:26.822032+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[31149]: audit 2024-03-07T03:15:26.840519+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:27 smithi148 bash[31149]: audit 2024-03-07T03:15:27.006532+0000 mon.a (mon.0) 1055 : audit 0 from='client.? 172.21.15.148:0/1036522695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:28.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:27 smithi195 bash[23221]: cephadm 2024-03-07T03:15:26.383911+0000 mgr.x (mgr.24580) 15 : cephadm 1 Adjusting osd_memory_target on smithi148 to 2524M 2024-03-07T03:15:28.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:27 smithi195 bash[23221]: cluster 2024-03-07T03:15:26.597382+0000 mon.a (mon.0) 1052 : cluster 0 osdmap e187: 8 total, 8 up, 8 in 2024-03-07T03:15:28.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:27 smithi195 bash[23221]: audit 2024-03-07T03:15:26.822032+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:28.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:27 smithi195 bash[23221]: audit 2024-03-07T03:15:26.840519+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:28.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:27 smithi195 bash[23221]: audit 2024-03-07T03:15:27.006532+0000 mon.a (mon.0) 1055 : audit 0 from='client.? 172.21.15.148:0/1036522695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:29.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:28 smithi148 bash[31149]: cluster 2024-03-07T03:15:27.039830+0000 mgr.x (mgr.24580) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 511 B/s wr, 3 op/s 2024-03-07T03:15:29.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:28 smithi148 bash[31149]: cluster 2024-03-07T03:15:28.326318+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2024-03-07T03:15:29.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:28 smithi148 bash[21362]: cluster 2024-03-07T03:15:27.039830+0000 mgr.x (mgr.24580) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 511 B/s wr, 3 op/s 2024-03-07T03:15:29.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:28 smithi148 bash[21362]: cluster 2024-03-07T03:15:28.326318+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2024-03-07T03:15:29.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:28 smithi195 bash[23221]: cluster 2024-03-07T03:15:27.039830+0000 mgr.x (mgr.24580) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 511 B/s wr, 3 op/s 2024-03-07T03:15:29.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:28 smithi195 bash[23221]: cluster 2024-03-07T03:15:28.326318+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e188: 8 total, 8 up, 8 in 2024-03-07T03:15:30.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:30 smithi148 bash[21362]: cluster 2024-03-07T03:15:29.040611+0000 mgr.x (mgr.24580) 17 : cluster 0 pgmap v29: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 11 KiB/s wr, 187 op/s 2024-03-07T03:15:30.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:30 smithi148 bash[21362]: cluster 2024-03-07T03:15:29.350700+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2024-03-07T03:15:30.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:30 smithi148 bash[21362]: audit 2024-03-07T03:15:29.465690+0000 mon.a (mon.0) 1058 : audit 0 from='client.? 172.21.15.148:0/3573333319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:30.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:30 smithi148 bash[31149]: cluster 2024-03-07T03:15:29.040611+0000 mgr.x (mgr.24580) 17 : cluster 0 pgmap v29: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 11 KiB/s wr, 187 op/s 2024-03-07T03:15:30.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:30 smithi148 bash[31149]: cluster 2024-03-07T03:15:29.350700+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2024-03-07T03:15:30.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:30 smithi148 bash[31149]: audit 2024-03-07T03:15:29.465690+0000 mon.a (mon.0) 1058 : audit 0 from='client.? 172.21.15.148:0/3573333319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:30.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:30 smithi195 bash[23221]: cluster 2024-03-07T03:15:29.040611+0000 mgr.x (mgr.24580) 17 : cluster 0 pgmap v29: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 11 KiB/s wr, 187 op/s 2024-03-07T03:15:30.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:30 smithi195 bash[23221]: cluster 2024-03-07T03:15:29.350700+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e189: 8 total, 8 up, 8 in 2024-03-07T03:15:30.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:30 smithi195 bash[23221]: audit 2024-03-07T03:15:29.465690+0000 mon.a (mon.0) 1058 : audit 0 from='client.? 172.21.15.148:0/3573333319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:30.820 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (13017 ms) 2024-03-07T03:15:30.820 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-03-07T03:15:30.903 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:30.933 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:15:31.041 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:15:31.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:31 smithi148 bash[21362]: cluster 2024-03-07T03:15:30.363216+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2024-03-07T03:15:31.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:31 smithi148 bash[31149]: cluster 2024-03-07T03:15:30.363216+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2024-03-07T03:15:31.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:31 smithi195 bash[23221]: cluster 2024-03-07T03:15:30.363216+0000 mon.a (mon.0) 1059 : cluster 0 osdmap e190: 8 total, 8 up, 8 in 2024-03-07T03:15:32.392 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:15:32.429 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:15:32.430 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:15:32.431 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:15:32.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:32 smithi148 bash[21362]: cluster 2024-03-07T03:15:31.042044+0000 mgr.x (mgr.24580) 18 : cluster 0 pgmap v32: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 11 KiB/s wr, 187 op/s 2024-03-07T03:15:32.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:32 smithi148 bash[21362]: cluster 2024-03-07T03:15:31.375778+0000 mon.a (mon.0) 1060 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2024-03-07T03:15:32.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:32 smithi148 bash[21362]: audit 2024-03-07T03:15:31.906828+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.148:0/3110687846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:32.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:32 smithi148 bash[31149]: cluster 2024-03-07T03:15:31.042044+0000 mgr.x (mgr.24580) 18 : cluster 0 pgmap v32: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 11 KiB/s wr, 187 op/s 2024-03-07T03:15:32.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:32 smithi148 bash[31149]: cluster 2024-03-07T03:15:31.375778+0000 mon.a (mon.0) 1060 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2024-03-07T03:15:32.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:32 smithi148 bash[31149]: audit 2024-03-07T03:15:31.906828+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.148:0/3110687846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:32.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:32 smithi195 bash[23221]: cluster 2024-03-07T03:15:31.042044+0000 mgr.x (mgr.24580) 18 : cluster 0 pgmap v32: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 18 MiB/s rd, 11 KiB/s wr, 187 op/s 2024-03-07T03:15:32.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:32 smithi195 bash[23221]: cluster 2024-03-07T03:15:31.375778+0000 mon.a (mon.0) 1060 : cluster 0 osdmap e191: 8 total, 8 up, 8 in 2024-03-07T03:15:32.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:32 smithi195 bash[23221]: audit 2024-03-07T03:15:31.906828+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.148:0/3110687846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:33.637 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:33 smithi195 bash[23221]: cluster 2024-03-07T03:15:32.383317+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2024-03-07T03:15:33.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:33 smithi148 bash[21362]: cluster 2024-03-07T03:15:32.383317+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2024-03-07T03:15:33.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:33 smithi148 bash[31149]: cluster 2024-03-07T03:15:32.383317+0000 mon.a (mon.0) 1061 : cluster 0 osdmap e192: 8 total, 8 up, 8 in 2024-03-07T03:15:34.406 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:15:34.413 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:15:34.413 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:15:34.434 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:15:34.469 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:15:34.629 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:34 smithi148 bash[21362]: cluster 2024-03-07T03:15:33.042976+0000 mgr.x (mgr.24580) 19 : cluster 0 pgmap v35: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 441 op/s 2024-03-07T03:15:34.630 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:34 smithi148 bash[21362]: cluster 2024-03-07T03:15:33.400452+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2024-03-07T03:15:34.630 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:34 smithi148 bash[21362]: audit 2024-03-07T03:15:34.342869+0000 mon.a (mon.0) 1063 : audit 0 from='client.? 172.21.15.148:0/3037027295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:34.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:34 smithi148 bash[31149]: cluster 2024-03-07T03:15:33.042976+0000 mgr.x (mgr.24580) 19 : cluster 0 pgmap v35: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 441 op/s 2024-03-07T03:15:34.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:34 smithi148 bash[31149]: cluster 2024-03-07T03:15:33.400452+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2024-03-07T03:15:34.630 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:34 smithi148 bash[31149]: audit 2024-03-07T03:15:34.342869+0000 mon.a (mon.0) 1063 : audit 0 from='client.? 172.21.15.148:0/3037027295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:34.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:34 smithi195 bash[23221]: cluster 2024-03-07T03:15:33.042976+0000 mgr.x (mgr.24580) 19 : cluster 0 pgmap v35: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 441 op/s 2024-03-07T03:15:34.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:34 smithi195 bash[23221]: cluster 2024-03-07T03:15:33.400452+0000 mon.a (mon.0) 1062 : cluster 0 osdmap e193: 8 total, 8 up, 8 in 2024-03-07T03:15:34.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:34 smithi195 bash[23221]: audit 2024-03-07T03:15:34.342869+0000 mon.a (mon.0) 1063 : audit 0 from='client.? 172.21.15.148:0/3037027295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:35.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:34 smithi148 bash[21615]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:15:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:15:35.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:35 smithi148 bash[21362]: cluster 2024-03-07T03:15:34.407573+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2024-03-07T03:15:35.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:35 smithi148 bash[31149]: cluster 2024-03-07T03:15:34.407573+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2024-03-07T03:15:35.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:35 smithi195 bash[23221]: cluster 2024-03-07T03:15:34.407573+0000 mon.a (mon.0) 1064 : cluster 0 osdmap e194: 8 total, 8 up, 8 in 2024-03-07T03:15:36.420 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:36.456 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:15:36.467 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:15:36.770 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:36 smithi195 bash[23221]: cluster 2024-03-07T03:15:35.043758+0000 mgr.x (mgr.24580) 20 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 441 op/s 2024-03-07T03:15:36.770 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:36 smithi195 bash[23221]: cluster 2024-03-07T03:15:35.416928+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2024-03-07T03:15:36.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:36 smithi148 bash[31149]: cluster 2024-03-07T03:15:35.043758+0000 mgr.x (mgr.24580) 20 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 441 op/s 2024-03-07T03:15:36.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:36 smithi148 bash[31149]: cluster 2024-03-07T03:15:35.416928+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2024-03-07T03:15:36.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:36 smithi148 bash[21362]: cluster 2024-03-07T03:15:35.043758+0000 mgr.x (mgr.24580) 20 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 577 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 46 MiB/s rd, 10 MiB/s wr, 441 op/s 2024-03-07T03:15:36.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:36 smithi148 bash[21362]: cluster 2024-03-07T03:15:35.416928+0000 mon.a (mon.0) 1065 : cluster 0 osdmap e195: 8 total, 8 up, 8 in 2024-03-07T03:15:37.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[21362]: cluster 2024-03-07T03:15:36.419719+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2024-03-07T03:15:37.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[21362]: audit 2024-03-07T03:15:36.805303+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.148:0/2411855471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:37.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[21362]: audit 2024-03-07T03:15:37.398960+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:37.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[21362]: audit 2024-03-07T03:15:37.415644+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[21362]: audit 2024-03-07T03:15:37.417829+0000 mon.b (mon.1) 174 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[21362]: audit 2024-03-07T03:15:37.421406+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[31149]: cluster 2024-03-07T03:15:36.419719+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[31149]: audit 2024-03-07T03:15:36.805303+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.148:0/2411855471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[31149]: audit 2024-03-07T03:15:37.398960+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[31149]: audit 2024-03-07T03:15:37.415644+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[31149]: audit 2024-03-07T03:15:37.417829+0000 mon.b (mon.1) 174 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:37.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:37 smithi148 bash[31149]: audit 2024-03-07T03:15:37.421406+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:37 smithi195 bash[23221]: cluster 2024-03-07T03:15:36.419719+0000 mon.a (mon.0) 1066 : cluster 0 osdmap e196: 8 total, 8 up, 8 in 2024-03-07T03:15:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:37 smithi195 bash[23221]: audit 2024-03-07T03:15:36.805303+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.148:0/2411855471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:37 smithi195 bash[23221]: audit 2024-03-07T03:15:37.398960+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:37 smithi195 bash[23221]: audit 2024-03-07T03:15:37.415644+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:37 smithi195 bash[23221]: audit 2024-03-07T03:15:37.417829+0000 mon.b (mon.1) 174 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:37 smithi195 bash[23221]: audit 2024-03-07T03:15:37.421406+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cluster 2024-03-07T03:15:37.045559+0000 mgr.x (mgr.24580) 21 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 597 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 4.0 MiB/s wr, 74 op/s 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.420973+0000 mon.b (mon.1) 175 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.431606+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cluster 2024-03-07T03:15:37.444587+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.445549+0000 mon.b (mon.1) 176 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.449310+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.449440+0000 mon.b (mon.1) 177 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cephadm 2024-03-07T03:15:37.452735+0000 mgr.x (mgr.24580) 22 : cephadm 1 Adjusting osd_memory_target on smithi195 to 2524M 2024-03-07T03:15:38.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.453746+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.467887+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.467960+0000 mon.b (mon.1) 178 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:37.469712+0000 mon.b (mon.1) 179 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cephadm 2024-03-07T03:15:37.471952+0000 mgr.x (mgr.24580) 23 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cephadm 2024-03-07T03:15:37.472391+0000 mgr.x (mgr.24580) 24 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cephadm 2024-03-07T03:15:37.737505+0000 mgr.x (mgr.24580) 25 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cephadm 2024-03-07T03:15:37.739532+0000 mgr.x (mgr.24580) 26 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cephadm 2024-03-07T03:15:37.902147+0000 mgr.x (mgr.24580) 27 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cephadm 2024-03-07T03:15:37.904622+0000 mgr.x (mgr.24580) 28 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:38.243548+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:38.252546+0000 mon.a (mon.0) 1076 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:38.263665+0000 mon.a (mon.0) 1077 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:38.272901+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: audit 2024-03-07T03:15:38.283380+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[21362]: cluster 2024-03-07T03:15:38.331706+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cluster 2024-03-07T03:15:37.045559+0000 mgr.x (mgr.24580) 21 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 597 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 4.0 MiB/s wr, 74 op/s 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.420973+0000 mon.b (mon.1) 175 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.431606+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cluster 2024-03-07T03:15:37.444587+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.445549+0000 mon.b (mon.1) 176 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.449310+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.449440+0000 mon.b (mon.1) 177 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cephadm 2024-03-07T03:15:37.452735+0000 mgr.x (mgr.24580) 22 : cephadm 1 Adjusting osd_memory_target on smithi195 to 2524M 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.453746+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.467887+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.467960+0000 mon.b (mon.1) 178 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:37.469712+0000 mon.b (mon.1) 179 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cephadm 2024-03-07T03:15:37.471952+0000 mgr.x (mgr.24580) 23 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cephadm 2024-03-07T03:15:37.472391+0000 mgr.x (mgr.24580) 24 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cephadm 2024-03-07T03:15:37.737505+0000 mgr.x (mgr.24580) 25 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cephadm 2024-03-07T03:15:37.739532+0000 mgr.x (mgr.24580) 26 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cephadm 2024-03-07T03:15:37.902147+0000 mgr.x (mgr.24580) 27 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cephadm 2024-03-07T03:15:37.904622+0000 mgr.x (mgr.24580) 28 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:38.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:38.243548+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:38.252546+0000 mon.a (mon.0) 1076 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:38.263665+0000 mon.a (mon.0) 1077 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:38.272901+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: audit 2024-03-07T03:15:38.283380+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:38 smithi148 bash[31149]: cluster 2024-03-07T03:15:38.331706+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cluster 2024-03-07T03:15:37.045559+0000 mgr.x (mgr.24580) 21 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 597 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 43 KiB/s rd, 4.0 MiB/s wr, 74 op/s 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.420973+0000 mon.b (mon.1) 175 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.431606+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cluster 2024-03-07T03:15:37.444587+0000 mon.a (mon.0) 1071 : cluster 0 osdmap e197: 8 total, 8 up, 8 in 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.445549+0000 mon.b (mon.1) 176 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.449310+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.449440+0000 mon.b (mon.1) 177 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cephadm 2024-03-07T03:15:37.452735+0000 mgr.x (mgr.24580) 22 : cephadm 1 Adjusting osd_memory_target on smithi195 to 2524M 2024-03-07T03:15:38.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.453746+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.467887+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.467960+0000 mon.b (mon.1) 178 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:37.469712+0000 mon.b (mon.1) 179 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cephadm 2024-03-07T03:15:37.471952+0000 mgr.x (mgr.24580) 23 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cephadm 2024-03-07T03:15:37.472391+0000 mgr.x (mgr.24580) 24 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cephadm 2024-03-07T03:15:37.737505+0000 mgr.x (mgr.24580) 25 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cephadm 2024-03-07T03:15:37.739532+0000 mgr.x (mgr.24580) 26 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cephadm 2024-03-07T03:15:37.902147+0000 mgr.x (mgr.24580) 27 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cephadm 2024-03-07T03:15:37.904622+0000 mgr.x (mgr.24580) 28 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:38.243548+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:38.252546+0000 mon.a (mon.0) 1076 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:38.263665+0000 mon.a (mon.0) 1077 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:38.272901+0000 mon.a (mon.0) 1078 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: audit 2024-03-07T03:15:38.283380+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:38 smithi195 bash[23221]: cluster 2024-03-07T03:15:38.331706+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e198: 8 total, 8 up, 8 in 2024-03-07T03:15:39.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[21362]: cephadm 2024-03-07T03:15:38.066771+0000 mgr.x (mgr.24580) 29 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:39.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[21362]: cephadm 2024-03-07T03:15:38.070834+0000 mgr.x (mgr.24580) 30 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[21362]: cephadm 2024-03-07T03:15:38.363202+0000 mgr.x (mgr.24580) 31 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[21362]: cephadm 2024-03-07T03:15:38.662260+0000 mgr.x (mgr.24580) 32 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[21362]: audit 2024-03-07T03:15:39.234351+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.148:0/3770079061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[21362]: cluster 2024-03-07T03:15:39.344131+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[31149]: cephadm 2024-03-07T03:15:38.066771+0000 mgr.x (mgr.24580) 29 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[31149]: cephadm 2024-03-07T03:15:38.070834+0000 mgr.x (mgr.24580) 30 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[31149]: cephadm 2024-03-07T03:15:38.363202+0000 mgr.x (mgr.24580) 31 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[31149]: cephadm 2024-03-07T03:15:38.662260+0000 mgr.x (mgr.24580) 32 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[31149]: audit 2024-03-07T03:15:39.234351+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.148:0/3770079061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:39.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:39 smithi148 bash[31149]: cluster 2024-03-07T03:15:39.344131+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2024-03-07T03:15:39.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:39 smithi195 bash[23221]: cephadm 2024-03-07T03:15:38.066771+0000 mgr.x (mgr.24580) 29 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:39.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:39 smithi195 bash[23221]: cephadm 2024-03-07T03:15:38.070834+0000 mgr.x (mgr.24580) 30 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:15:39.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:39 smithi195 bash[23221]: cephadm 2024-03-07T03:15:38.363202+0000 mgr.x (mgr.24580) 31 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:15:39.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:39 smithi195 bash[23221]: cephadm 2024-03-07T03:15:38.662260+0000 mgr.x (mgr.24580) 32 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:15:39.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:39 smithi195 bash[23221]: audit 2024-03-07T03:15:39.234351+0000 mon.c (mon.2) 57 : audit 0 from='client.? 172.21.15.148:0/3770079061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:39.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:39 smithi195 bash[23221]: cluster 2024-03-07T03:15:39.344131+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e199: 8 total, 8 up, 8 in 2024-03-07T03:15:40.238 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:15:40.265 INFO:tasks.workunit.client.0.smithi148.stdout:write: * 83886080~1000 2024-03-07T03:15:40.328 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:15:40.558 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:15:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:40 smithi195 bash[23221]: cluster 2024-03-07T03:15:39.046143+0000 mgr.x (mgr.24580) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:40.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:40 smithi148 bash[21362]: cluster 2024-03-07T03:15:39.046143+0000 mgr.x (mgr.24580) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:40.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:40 smithi148 bash[31149]: cluster 2024-03-07T03:15:39.046143+0000 mgr.x (mgr.24580) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-03-07T03:15:41.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:41 smithi195 bash[23221]: audit 2024-03-07T03:15:41.156739+0000 mon.b (mon.1) 180 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:41.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:41 smithi148 bash[31149]: audit 2024-03-07T03:15:41.156739+0000 mon.b (mon.1) 180 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:41.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:41 smithi148 bash[21362]: audit 2024-03-07T03:15:41.156739+0000 mon.b (mon.1) 180 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:42.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:42 smithi148 bash[21362]: cluster 2024-03-07T03:15:41.047254+0000 mgr.x (mgr.24580) 34 : cluster 0 pgmap v46: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 121 KiB/s rd, 6.9 MiB/s wr, 174 op/s 2024-03-07T03:15:42.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:42 smithi148 bash[21362]: cluster 2024-03-07T03:15:41.463581+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2024-03-07T03:15:42.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:42 smithi148 bash[21362]: audit 2024-03-07T03:15:41.692335+0000 mon.a (mon.0) 1083 : audit 0 from='client.? 172.21.15.148:0/1302836877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:42.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:42 smithi148 bash[31149]: cluster 2024-03-07T03:15:41.047254+0000 mgr.x (mgr.24580) 34 : cluster 0 pgmap v46: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 121 KiB/s rd, 6.9 MiB/s wr, 174 op/s 2024-03-07T03:15:42.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:42 smithi148 bash[31149]: cluster 2024-03-07T03:15:41.463581+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2024-03-07T03:15:42.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:42 smithi148 bash[31149]: audit 2024-03-07T03:15:41.692335+0000 mon.a (mon.0) 1083 : audit 0 from='client.? 172.21.15.148:0/1302836877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:42.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:42 smithi195 bash[23221]: cluster 2024-03-07T03:15:41.047254+0000 mgr.x (mgr.24580) 34 : cluster 0 pgmap v46: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 121 KiB/s rd, 6.9 MiB/s wr, 174 op/s 2024-03-07T03:15:42.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:42 smithi195 bash[23221]: cluster 2024-03-07T03:15:41.463581+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e200: 8 total, 8 up, 8 in 2024-03-07T03:15:42.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:42 smithi195 bash[23221]: audit 2024-03-07T03:15:41.692335+0000 mon.a (mon.0) 1083 : audit 0 from='client.? 172.21.15.148:0/1302836877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 systemd[1]: Stopping Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.135Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.136Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.136Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:15:43.304 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[49720]: ts=2024-03-07T03:15:43.136Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:15:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[23221]: cluster 2024-03-07T03:15:42.472890+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2024-03-07T03:15:43.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:43 smithi148 bash[21362]: cluster 2024-03-07T03:15:42.472890+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2024-03-07T03:15:43.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:43 smithi148 bash[31149]: cluster 2024-03-07T03:15:42.472890+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e201: 8 total, 8 up, 8 in 2024-03-07T03:15:44.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[57718]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:15:44.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 bash[57777]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:15:44.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-07T03:15:44.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 systemd[1]: Stopped Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:15:44.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:43 smithi195 systemd[1]: Started Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:15:44.065 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (13245 ms) 2024-03-07T03:15:44.065 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-03-07T03:15:44.134 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:44.158 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 41943040~10 2024-03-07T03:15:44.176 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:15:44.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: cluster 2024-03-07T03:15:43.047750+0000 mgr.x (mgr.24580) 35 : cluster 0 pgmap v49: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 5.1 MiB/s wr, 360 op/s 2024-03-07T03:15:44.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: cluster 2024-03-07T03:15:43.483076+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2024-03-07T03:15:44.773 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: audit 2024-03-07T03:15:43.869353+0000 mon.a (mon.0) 1086 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:44.774 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: audit 2024-03-07T03:15:43.878510+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:44.774 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: audit 2024-03-07T03:15:43.917611+0000 mon.b (mon.1) 181 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:15:44.774 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: audit 2024-03-07T03:15:43.918284+0000 mgr.x (mgr.24580) 36 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:15:44.774 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: audit 2024-03-07T03:15:43.925791+0000 mon.b (mon.1) 182 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:44.774 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: cephadm 2024-03-07T03:15:43.927092+0000 mgr.x (mgr.24580) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:15:44.774 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: cephadm 2024-03-07T03:15:43.927712+0000 mgr.x (mgr.24580) 38 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:15:44.774 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[23221]: audit 2024-03-07T03:15:44.205211+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.148:0/212909749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:44.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: cluster 2024-03-07T03:15:43.047750+0000 mgr.x (mgr.24580) 35 : cluster 0 pgmap v49: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 5.1 MiB/s wr, 360 op/s 2024-03-07T03:15:44.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: cluster 2024-03-07T03:15:43.483076+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2024-03-07T03:15:44.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: audit 2024-03-07T03:15:43.869353+0000 mon.a (mon.0) 1086 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:44.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: audit 2024-03-07T03:15:43.878510+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:44.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: audit 2024-03-07T03:15:43.917611+0000 mon.b (mon.1) 181 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: audit 2024-03-07T03:15:43.918284+0000 mgr.x (mgr.24580) 36 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: audit 2024-03-07T03:15:43.925791+0000 mon.b (mon.1) 182 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: cephadm 2024-03-07T03:15:43.927092+0000 mgr.x (mgr.24580) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: cephadm 2024-03-07T03:15:43.927712+0000 mgr.x (mgr.24580) 38 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[21362]: audit 2024-03-07T03:15:44.205211+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.148:0/212909749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: cluster 2024-03-07T03:15:43.047750+0000 mgr.x (mgr.24580) 35 : cluster 0 pgmap v49: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 5.1 MiB/s wr, 360 op/s 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: cluster 2024-03-07T03:15:43.483076+0000 mon.a (mon.0) 1085 : cluster 0 osdmap e202: 8 total, 8 up, 8 in 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: audit 2024-03-07T03:15:43.869353+0000 mon.a (mon.0) 1086 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: audit 2024-03-07T03:15:43.878510+0000 mon.a (mon.0) 1087 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: audit 2024-03-07T03:15:43.917611+0000 mon.b (mon.1) 181 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: audit 2024-03-07T03:15:43.918284+0000 mgr.x (mgr.24580) 36 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: audit 2024-03-07T03:15:43.925791+0000 mon.b (mon.1) 182 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: cephadm 2024-03-07T03:15:43.927092+0000 mgr.x (mgr.24580) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:15:44.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: cephadm 2024-03-07T03:15:43.927712+0000 mgr.x (mgr.24580) 38 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-03-07T03:15:44.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:44 smithi148 bash[31149]: audit 2024-03-07T03:15:44.205211+0000 mon.c (mon.2) 58 : audit 0 from='client.? 172.21.15.148:0/212909749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:45.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.901Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:15:45.063 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.901Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.901Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi195 (none))" 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.901Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.901Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.903Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.903Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.904Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.905Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.908Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.908Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.186µs 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.908Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.910Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.921Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-03-07T03:15:45.064 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.922Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-03-07T03:15:45.065 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.922Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=32.656µs wal_replay_duration=13.781614ms wbl_replay_duration=105ns total_replay_duration=13.845326ms 2024-03-07T03:15:45.065 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.924Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:15:45.065 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.924Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:15:45.065 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.924Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:15:45.065 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.949Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.490395ms db_storage=1.327µs remote_storage=1.36µs web_handler=412ns query_engine=640ns scrape=341.287µs scrape_sd=108.706µs notify=10.388µs notify_sd=15.493µs rules=24.614222ms tracing=7.731µs 2024-03-07T03:15:45.065 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.949Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:15:45.065 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:44 smithi195 bash[57805]: ts=2024-03-07T03:15:44.949Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:15:45.899 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:15:45.954 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:15:46.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:45 smithi148 bash[21362]: cluster 2024-03-07T03:15:44.885566+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2024-03-07T03:15:46.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:45 smithi148 bash[31149]: cluster 2024-03-07T03:15:44.885566+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2024-03-07T03:15:46.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:45 smithi195 bash[23221]: cluster 2024-03-07T03:15:44.885566+0000 mon.a (mon.0) 1088 : cluster 0 osdmap e203: 8 total, 8 up, 8 in 2024-03-07T03:15:47.263 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:47.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: cluster 2024-03-07T03:15:45.048473+0000 mgr.x (mgr.24580) 39 : cluster 0 pgmap v52: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 426 op/s 2024-03-07T03:15:47.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: cluster 2024-03-07T03:15:45.892476+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2024-03-07T03:15:47.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: audit 2024-03-07T03:15:46.407979+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:47.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: audit 2024-03-07T03:15:46.409999+0000 mon.b (mon.1) 183 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:15:47.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: audit 2024-03-07T03:15:46.412808+0000 mon.b (mon.1) 184 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: audit 2024-03-07T03:15:46.413252+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: audit 2024-03-07T03:15:46.414989+0000 mon.b (mon.1) 185 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[21362]: audit 2024-03-07T03:15:46.649039+0000 mon.a (mon.0) 1092 : audit 0 from='client.? 172.21.15.148:0/3205626078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: cluster 2024-03-07T03:15:45.048473+0000 mgr.x (mgr.24580) 39 : cluster 0 pgmap v52: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 426 op/s 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: cluster 2024-03-07T03:15:45.892476+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: audit 2024-03-07T03:15:46.407979+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: audit 2024-03-07T03:15:46.409999+0000 mon.b (mon.1) 183 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: audit 2024-03-07T03:15:46.412808+0000 mon.b (mon.1) 184 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: audit 2024-03-07T03:15:46.413252+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: audit 2024-03-07T03:15:46.414989+0000 mon.b (mon.1) 185 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:47.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:46 smithi148 bash[31149]: audit 2024-03-07T03:15:46.649039+0000 mon.a (mon.0) 1092 : audit 0 from='client.? 172.21.15.148:0/3205626078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:47.293 INFO:tasks.workunit.client.0.smithi148.stdout:write: Y 41943040~5 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: cluster 2024-03-07T03:15:45.048473+0000 mgr.x (mgr.24580) 39 : cluster 0 pgmap v52: 137 pgs: 137 active+clean; 637 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 426 op/s 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: cluster 2024-03-07T03:15:45.892476+0000 mon.a (mon.0) 1089 : cluster 0 osdmap e204: 8 total, 8 up, 8 in 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: audit 2024-03-07T03:15:46.407979+0000 mon.a (mon.0) 1090 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: audit 2024-03-07T03:15:46.409999+0000 mon.b (mon.1) 183 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: audit 2024-03-07T03:15:46.412808+0000 mon.b (mon.1) 184 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: audit 2024-03-07T03:15:46.413252+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: audit 2024-03-07T03:15:46.414989+0000 mon.b (mon.1) 185 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:15:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:46 smithi195 bash[23221]: audit 2024-03-07T03:15:46.649039+0000 mon.a (mon.0) 1092 : audit 0 from='client.? 172.21.15.148:0/3205626078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:47.536 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:15:47.738 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:15:48.100 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (4035 ms) 2024-03-07T03:15:48.100 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-03-07T03:15:48.184 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:48.214 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:15:48.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:47 smithi148 bash[21362]: cephadm 2024-03-07T03:15:46.397733+0000 mgr.x (mgr.24580) 40 : cephadm 1 Upgrade: Updating mgr.y 2024-03-07T03:15:48.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:47 smithi148 bash[21362]: cephadm 2024-03-07T03:15:46.417216+0000 mgr.x (mgr.24580) 41 : cephadm 1 Deploying daemon mgr.y on smithi148 2024-03-07T03:15:48.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:47 smithi148 bash[21362]: cluster 2024-03-07T03:15:46.897834+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2024-03-07T03:15:48.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:47 smithi148 bash[31149]: cephadm 2024-03-07T03:15:46.397733+0000 mgr.x (mgr.24580) 40 : cephadm 1 Upgrade: Updating mgr.y 2024-03-07T03:15:48.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:47 smithi148 bash[31149]: cephadm 2024-03-07T03:15:46.417216+0000 mgr.x (mgr.24580) 41 : cephadm 1 Deploying daemon mgr.y on smithi148 2024-03-07T03:15:48.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:47 smithi148 bash[31149]: cluster 2024-03-07T03:15:46.897834+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2024-03-07T03:15:48.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:47 smithi195 bash[23221]: cephadm 2024-03-07T03:15:46.397733+0000 mgr.x (mgr.24580) 40 : cephadm 1 Upgrade: Updating mgr.y 2024-03-07T03:15:48.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:47 smithi195 bash[23221]: cephadm 2024-03-07T03:15:46.417216+0000 mgr.x (mgr.24580) 41 : cephadm 1 Deploying daemon mgr.y on smithi148 2024-03-07T03:15:48.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:47 smithi195 bash[23221]: cluster 2024-03-07T03:15:46.897834+0000 mon.a (mon.0) 1093 : cluster 0 osdmap e205: 8 total, 8 up, 8 in 2024-03-07T03:15:48.351 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:15:48.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.788 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.789 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.789 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: Stopping Ceph mgr.y for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:15:48.790 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.790 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:48.790 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:15:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:48 smithi148 bash[21362]: cluster 2024-03-07T03:15:47.050026+0000 mgr.x (mgr.24580) 42 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 16 KiB/s wr, 207 op/s 2024-03-07T03:15:49.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:48 smithi148 bash[21362]: cluster 2024-03-07T03:15:47.907873+0000 mon.a (mon.0) 1094 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2024-03-07T03:15:49.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:48 smithi148 bash[21362]: cluster 2024-03-07T03:15:48.340266+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2024-03-07T03:15:49.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:48 smithi148 bash[31149]: cluster 2024-03-07T03:15:47.050026+0000 mgr.x (mgr.24580) 42 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 16 KiB/s wr, 207 op/s 2024-03-07T03:15:49.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:48 smithi148 bash[31149]: cluster 2024-03-07T03:15:47.907873+0000 mon.a (mon.0) 1094 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2024-03-07T03:15:49.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:48 smithi148 bash[31149]: cluster 2024-03-07T03:15:48.340266+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2024-03-07T03:15:49.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:48 smithi195 bash[23221]: cluster 2024-03-07T03:15:47.050026+0000 mgr.x (mgr.24580) 42 : cluster 0 pgmap v55: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 16 KiB/s wr, 207 op/s 2024-03-07T03:15:49.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:48 smithi195 bash[23221]: cluster 2024-03-07T03:15:47.907873+0000 mon.a (mon.0) 1094 : cluster 0 osdmap e206: 8 total, 8 up, 8 in 2024-03-07T03:15:49.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:48 smithi195 bash[23221]: cluster 2024-03-07T03:15:48.340266+0000 mon.a (mon.0) 1095 : cluster 0 osdmap e207: 8 total, 8 up, 8 in 2024-03-07T03:15:49.646 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[72445]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mgr-y 2024-03-07T03:15:49.646 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:15:49.646 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[72528]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mgr-y 2024-03-07T03:15:49.646 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y.service: Failed with result 'exit-code'. 2024-03-07T03:15:49.646 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: Stopped Ceph mgr.y for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:15:49.899 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.899 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.899 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.900 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.900 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.901 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.901 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.901 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.901 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:15:49.901 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:49 smithi148 systemd[1]: Started Ceph mgr.y for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:15:50.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[21362]: audit 2024-03-07T03:15:49.090471+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.148:0/3384337528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:50.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[21362]: cluster 2024-03-07T03:15:49.351088+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2024-03-07T03:15:50.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[21362]: audit 2024-03-07T03:15:49.837980+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:50.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[21362]: audit 2024-03-07T03:15:49.862059+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:50.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[21362]: audit 2024-03-07T03:15:49.891279+0000 mon.b (mon.1) 186 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:50.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[31149]: audit 2024-03-07T03:15:49.090471+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.148:0/3384337528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:50.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[31149]: cluster 2024-03-07T03:15:49.351088+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2024-03-07T03:15:50.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[31149]: audit 2024-03-07T03:15:49.837980+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:50.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[31149]: audit 2024-03-07T03:15:49.862059+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:50.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:49 smithi148 bash[31149]: audit 2024-03-07T03:15:49.891279+0000 mon.b (mon.1) 186 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:50.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:49 smithi195 bash[23221]: audit 2024-03-07T03:15:49.090471+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.148:0/3384337528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:50.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:49 smithi195 bash[23221]: cluster 2024-03-07T03:15:49.351088+0000 mon.a (mon.0) 1096 : cluster 0 osdmap e208: 8 total, 8 up, 8 in 2024-03-07T03:15:50.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:49 smithi195 bash[23221]: audit 2024-03-07T03:15:49.837980+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:50.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:49 smithi195 bash[23221]: audit 2024-03-07T03:15:49.862059+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:50.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:49 smithi195 bash[23221]: audit 2024-03-07T03:15:49.891279+0000 mon.b (mon.1) 186 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:15:50.347 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:15:50.367 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:15:50.367 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:15:50.367 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:15:51.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:50 smithi148 bash[21362]: cluster 2024-03-07T03:15:49.050805+0000 mgr.x (mgr.24580) 43 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 16 KiB/s wr, 207 op/s 2024-03-07T03:15:51.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:50 smithi148 bash[21362]: cluster 2024-03-07T03:15:50.353482+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2024-03-07T03:15:51.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:50 smithi148 bash[72584]: debug 2024-03-07T03:15:50.881+0000 7fef9afd1700 1 -- 172.21.15.148:0/2013790108 <== mon.2 v2:172.21.15.148:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x556ef66dc5a0 con 0x556ef7460400 2024-03-07T03:15:51.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:51 smithi148 bash[72584]: debug 2024-03-07T03:15:51.029+0000 7fefa51d3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:15:51.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:50 smithi148 bash[31149]: cluster 2024-03-07T03:15:49.050805+0000 mgr.x (mgr.24580) 43 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 16 KiB/s wr, 207 op/s 2024-03-07T03:15:51.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:50 smithi148 bash[31149]: cluster 2024-03-07T03:15:50.353482+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2024-03-07T03:15:51.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:50 smithi195 bash[23221]: cluster 2024-03-07T03:15:49.050805+0000 mgr.x (mgr.24580) 43 : cluster 0 pgmap v58: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 16 KiB/s wr, 207 op/s 2024-03-07T03:15:51.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:50 smithi195 bash[23221]: cluster 2024-03-07T03:15:50.353482+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e209: 8 total, 8 up, 8 in 2024-03-07T03:15:52.373 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:15:52.385 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:15:52.385 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:15:52.403 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:15:52.449 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:15:52.650 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[21362]: cluster 2024-03-07T03:15:51.051707+0000 mgr.x (mgr.24580) 44 : cluster 0 pgmap v61: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:52.650 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[21362]: cluster 2024-03-07T03:15:51.359158+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2024-03-07T03:15:52.650 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[21362]: audit 2024-03-07T03:15:51.528556+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.148:0/2270421364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:52.651 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[31149]: cluster 2024-03-07T03:15:51.051707+0000 mgr.x (mgr.24580) 44 : cluster 0 pgmap v61: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:52.651 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[31149]: cluster 2024-03-07T03:15:51.359158+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2024-03-07T03:15:52.651 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[31149]: audit 2024-03-07T03:15:51.528556+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.148:0/2270421364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:52.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:52 smithi195 bash[23221]: cluster 2024-03-07T03:15:51.051707+0000 mgr.x (mgr.24580) 44 : cluster 0 pgmap v61: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:15:52.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:52 smithi195 bash[23221]: cluster 2024-03-07T03:15:51.359158+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e210: 8 total, 8 up, 8 in 2024-03-07T03:15:52.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:52 smithi195 bash[23221]: audit 2024-03-07T03:15:51.528556+0000 mon.c (mon.2) 60 : audit 0 from='client.? 172.21.15.148:0/2270421364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:53.011 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[72584]: debug 2024-03-07T03:15:52.661+0000 7fefa51d3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:15:53.011 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:52 smithi148 bash[72584]: debug 2024-03-07T03:15:52.769+0000 7fefa51d3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:15:53.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:53 smithi148 bash[72584]: debug 2024-03-07T03:15:53.021+0000 7fefa51d3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:15:53.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:53 smithi148 bash[21362]: cluster 2024-03-07T03:15:52.370438+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2024-03-07T03:15:53.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:53 smithi148 bash[31149]: cluster 2024-03-07T03:15:52.370438+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2024-03-07T03:15:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:53 smithi195 bash[23221]: cluster 2024-03-07T03:15:52.370438+0000 mon.a (mon.0) 1101 : cluster 0 osdmap e211: 8 total, 8 up, 8 in 2024-03-07T03:15:53.813 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:15:53 smithi195 bash[57805]: ts=2024-03-07T03:15:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:15:54.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:53 smithi148 bash[72584]: debug 2024-03-07T03:15:53.845+0000 7fefa51d3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:15:54.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:53 smithi148 bash[72584]: debug 2024-03-07T03:15:53.957+0000 7fefa51d3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:15:54.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[72584]: debug 2024-03-07T03:15:54.065+0000 7fefa51d3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:15:54.387 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:15:54.424 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:15:54.435 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:15:54.661 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[72584]: debug 2024-03-07T03:15:54.436+0000 7fefa51d3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:15:54.661 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[72584]: debug 2024-03-07T03:15:54.564+0000 7fefa51d3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:15:54.662 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[21362]: cluster 2024-03-07T03:15:53.052541+0000 mgr.x (mgr.24580) 45 : cluster 0 pgmap v64: 137 pgs: 137 active+clean; 649 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 8.0 MiB/s wr, 311 op/s 2024-03-07T03:15:54.662 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[21362]: cluster 2024-03-07T03:15:53.377978+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2024-03-07T03:15:54.662 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[21362]: audit 2024-03-07T03:15:53.969063+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.148:0/2699637297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:54.662 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[31149]: cluster 2024-03-07T03:15:53.052541+0000 mgr.x (mgr.24580) 45 : cluster 0 pgmap v64: 137 pgs: 137 active+clean; 649 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 8.0 MiB/s wr, 311 op/s 2024-03-07T03:15:54.662 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[31149]: cluster 2024-03-07T03:15:53.377978+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2024-03-07T03:15:54.662 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[31149]: audit 2024-03-07T03:15:53.969063+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.148:0/2699637297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:54.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:54 smithi195 bash[23221]: cluster 2024-03-07T03:15:53.052541+0000 mgr.x (mgr.24580) 45 : cluster 0 pgmap v64: 137 pgs: 137 active+clean; 649 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 8.0 MiB/s wr, 311 op/s 2024-03-07T03:15:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:54 smithi195 bash[23221]: cluster 2024-03-07T03:15:53.377978+0000 mon.a (mon.0) 1102 : cluster 0 osdmap e212: 8 total, 8 up, 8 in 2024-03-07T03:15:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:54 smithi195 bash[23221]: audit 2024-03-07T03:15:53.969063+0000 mon.c (mon.2) 61 : audit 0 from='client.? 172.21.15.148:0/2699637297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:55.026 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:54 smithi148 bash[72584]: debug 2024-03-07T03:15:54.672+0000 7fefa51d3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:15:55.286 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:55 smithi148 bash[72584]: debug 2024-03-07T03:15:55.036+0000 7fefa51d3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:15:55.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:55 smithi148 bash[21362]: cluster 2024-03-07T03:15:54.382859+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2024-03-07T03:15:55.538 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:55 smithi148 bash[31149]: cluster 2024-03-07T03:15:54.382859+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2024-03-07T03:15:55.538 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:55 smithi148 bash[72584]: debug 2024-03-07T03:15:55.296+0000 7fefa51d3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:15:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:55 smithi195 bash[23221]: cluster 2024-03-07T03:15:54.382859+0000 mon.a (mon.0) 1103 : cluster 0 osdmap e213: 8 total, 8 up, 8 in 2024-03-07T03:15:56.025 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:55 smithi148 bash[72584]: debug 2024-03-07T03:15:55.608+0000 7fefa51d3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:15:56.025 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:55 smithi148 bash[72584]: debug 2024-03-07T03:15:55.736+0000 7fefa51d3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:15:56.025 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[72584]: debug 2024-03-07T03:15:56.036+0000 7fefa51d3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:15:56.738 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[72584]: debug 2024-03-07T03:15:56.308+0000 7fefa51d3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:56.738 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[31149]: cluster 2024-03-07T03:15:55.053055+0000 mgr.x (mgr.24580) 46 : cluster 0 pgmap v67: 137 pgs: 137 active+clean; 649 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 8.0 MiB/s wr, 311 op/s 2024-03-07T03:15:56.738 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[31149]: cluster 2024-03-07T03:15:55.394481+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2024-03-07T03:15:56.738 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[31149]: audit 2024-03-07T03:15:55.735559+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[31149]: audit 2024-03-07T03:15:55.744345+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[31149]: audit 2024-03-07T03:15:56.041184+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[31149]: audit 2024-03-07T03:15:56.050878+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[31149]: audit 2024-03-07T03:15:56.157106+0000 mon.b (mon.1) 187 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[21362]: cluster 2024-03-07T03:15:55.053055+0000 mgr.x (mgr.24580) 46 : cluster 0 pgmap v67: 137 pgs: 137 active+clean; 649 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 8.0 MiB/s wr, 311 op/s 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[21362]: cluster 2024-03-07T03:15:55.394481+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[21362]: audit 2024-03-07T03:15:55.735559+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[21362]: audit 2024-03-07T03:15:55.744345+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[21362]: audit 2024-03-07T03:15:56.041184+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[21362]: audit 2024-03-07T03:15:56.050878+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[21362]: audit 2024-03-07T03:15:56.157106+0000 mon.b (mon.1) 187 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:56 smithi195 bash[23221]: cluster 2024-03-07T03:15:55.053055+0000 mgr.x (mgr.24580) 46 : cluster 0 pgmap v67: 137 pgs: 137 active+clean; 649 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 238 KiB/s rd, 8.0 MiB/s wr, 311 op/s 2024-03-07T03:15:56.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:56 smithi195 bash[23221]: cluster 2024-03-07T03:15:55.394481+0000 mon.a (mon.0) 1104 : cluster 0 osdmap e214: 8 total, 8 up, 8 in 2024-03-07T03:15:56.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:56 smithi195 bash[23221]: audit 2024-03-07T03:15:55.735559+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:56 smithi195 bash[23221]: audit 2024-03-07T03:15:55.744345+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:56 smithi195 bash[23221]: audit 2024-03-07T03:15:56.041184+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:56 smithi195 bash[23221]: audit 2024-03-07T03:15:56.050878+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:56.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:56 smithi195 bash[23221]: audit 2024-03-07T03:15:56.157106+0000 mon.b (mon.1) 187 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:15:57.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[72584]: debug 2024-03-07T03:15:56.748+0000 7fefa51d3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:15:57.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:56 smithi148 bash[72584]: debug 2024-03-07T03:15:56.872+0000 7fefa51d3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:15:57.394 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[72584]: debug 2024-03-07T03:15:57.132+0000 7fefa51d3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:15:57.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[31149]: cluster 2024-03-07T03:15:56.404471+0000 mon.a (mon.0) 1109 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2024-03-07T03:15:57.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[31149]: audit 2024-03-07T03:15:56.410980+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.148:0/282114650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:57.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[31149]: audit 2024-03-07T03:15:56.920976+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:57.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[31149]: audit 2024-03-07T03:15:56.940954+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[21362]: cluster 2024-03-07T03:15:56.404471+0000 mon.a (mon.0) 1109 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2024-03-07T03:15:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[21362]: audit 2024-03-07T03:15:56.410980+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.148:0/282114650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[21362]: audit 2024-03-07T03:15:56.920976+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:57 smithi148 bash[21362]: audit 2024-03-07T03:15:56.940954+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:57 smithi195 bash[23221]: cluster 2024-03-07T03:15:56.404471+0000 mon.a (mon.0) 1109 : cluster 0 osdmap e215: 8 total, 8 up, 8 in 2024-03-07T03:15:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:57 smithi195 bash[23221]: audit 2024-03-07T03:15:56.410980+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.148:0/282114650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:57 smithi195 bash[23221]: audit 2024-03-07T03:15:56.920976+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:57.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:57 smithi195 bash[23221]: audit 2024-03-07T03:15:56.940954+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:15:58.426 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: after_prepare_snap 2024-03-07T03:15:58.746 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[72584]: debug 2024-03-07T03:15:58.300+0000 7fefa51d3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:15:58.746 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[72584]: debug 2024-03-07T03:15:58.448+0000 7fefa51d3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:15:58.747 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[21362]: cluster 2024-03-07T03:15:57.054609+0000 mgr.x (mgr.24580) 47 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-07T03:15:58.747 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[21362]: cluster 2024-03-07T03:15:57.404804+0000 mon.a (mon.0) 1112 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2024-03-07T03:15:58.747 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[21362]: cluster 2024-03-07T03:15:58.335213+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2024-03-07T03:15:58.747 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[31149]: cluster 2024-03-07T03:15:57.054609+0000 mgr.x (mgr.24580) 47 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-07T03:15:58.747 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[31149]: cluster 2024-03-07T03:15:57.404804+0000 mon.a (mon.0) 1112 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2024-03-07T03:15:58.747 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[31149]: cluster 2024-03-07T03:15:58.335213+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2024-03-07T03:15:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:58 smithi195 bash[23221]: cluster 2024-03-07T03:15:57.054609+0000 mgr.x (mgr.24580) 47 : cluster 0 pgmap v70: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-07T03:15:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:58 smithi195 bash[23221]: cluster 2024-03-07T03:15:57.404804+0000 mon.a (mon.0) 1112 : cluster 0 osdmap e216: 8 total, 8 up, 8 in 2024-03-07T03:15:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:58 smithi195 bash[23221]: cluster 2024-03-07T03:15:58.335213+0000 mon.a (mon.0) 1113 : cluster 0 osdmap e217: 8 total, 8 up, 8 in 2024-03-07T03:15:59.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[72584]: debug 2024-03-07T03:15:58.756+0000 7fefa51d3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:15:59.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:58 smithi148 bash[72584]: debug 2024-03-07T03:15:58.876+0000 7fefa51d3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:15:59.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:15:59 smithi195 bash[23221]: audit 2024-03-07T03:15:58.842453+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.148:0/2995981028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:59.563 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:15:59 smithi195 bash[51559]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:15:59] "GET /metrics HTTP/1.1" 200 38772 "" "Prometheus/2.43.0" 2024-03-07T03:15:59.617 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[31149]: audit 2024-03-07T03:15:58.842453+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.148:0/2995981028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:15:59.618 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[72584]: debug 2024-03-07T03:15:59.360+0000 7fefa51d3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:15:59.618 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[72584]: debug 2024-03-07T03:15:59.496+0000 7fefa51d3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:15:59.618 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[72584]: [07/Mar/2024:03:15:59] ENGINE Bus STARTING 2024-03-07T03:15:59.618 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[72584]: CherryPy Checker: 2024-03-07T03:15:59.618 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[21362]: audit 2024-03-07T03:15:58.842453+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.148:0/2995981028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:00.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[72584]: The Application mounted at '' has an empty config. 2024-03-07T03:16:00.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[72584]: [07/Mar/2024:03:15:59] ENGINE Serving on http://:::9283 2024-03-07T03:16:00.038 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:15:59 smithi148 bash[72584]: [07/Mar/2024:03:15:59] ENGINE Bus STARTED 2024-03-07T03:16:00.458 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:16:00.537 INFO:tasks.workunit.client.0.smithi148.stdout:write: * 0~1000 2024-03-07T03:16:00.552 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:16:00.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: cluster 2024-03-07T03:15:59.055418+0000 mgr.x (mgr.24580) 48 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-07T03:16:00.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: cluster 2024-03-07T03:15:59.424483+0000 mon.a (mon.0) 1114 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2024-03-07T03:16:00.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: cluster 2024-03-07T03:15:59.523913+0000 mon.a (mon.0) 1115 : cluster 0 Standby manager daemon y restarted 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: cluster 2024-03-07T03:15:59.524257+0000 mon.a (mon.0) 1116 : cluster 0 Standby manager daemon y started 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: audit 2024-03-07T03:15:59.527585+0000 mon.c (mon.2) 64 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: audit 2024-03-07T03:15:59.528971+0000 mon.c (mon.2) 65 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: audit 2024-03-07T03:15:59.532359+0000 mon.c (mon.2) 66 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[31149]: audit 2024-03-07T03:15:59.533303+0000 mon.c (mon.2) 67 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: cluster 2024-03-07T03:15:59.055418+0000 mgr.x (mgr.24580) 48 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: cluster 2024-03-07T03:15:59.424483+0000 mon.a (mon.0) 1114 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: cluster 2024-03-07T03:15:59.523913+0000 mon.a (mon.0) 1115 : cluster 0 Standby manager daemon y restarted 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: cluster 2024-03-07T03:15:59.524257+0000 mon.a (mon.0) 1116 : cluster 0 Standby manager daemon y started 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: audit 2024-03-07T03:15:59.527585+0000 mon.c (mon.2) 64 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:16:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: audit 2024-03-07T03:15:59.528971+0000 mon.c (mon.2) 65 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: audit 2024-03-07T03:15:59.532359+0000 mon.c (mon.2) 66 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:16:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:00 smithi148 bash[21362]: audit 2024-03-07T03:15:59.533303+0000 mon.c (mon.2) 67 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:00.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: cluster 2024-03-07T03:15:59.055418+0000 mgr.x (mgr.24580) 48 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 169 op/s 2024-03-07T03:16:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: cluster 2024-03-07T03:15:59.424483+0000 mon.a (mon.0) 1114 : cluster 0 osdmap e218: 8 total, 8 up, 8 in 2024-03-07T03:16:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: cluster 2024-03-07T03:15:59.523913+0000 mon.a (mon.0) 1115 : cluster 0 Standby manager daemon y restarted 2024-03-07T03:16:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: cluster 2024-03-07T03:15:59.524257+0000 mon.a (mon.0) 1116 : cluster 0 Standby manager daemon y started 2024-03-07T03:16:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: audit 2024-03-07T03:15:59.527585+0000 mon.c (mon.2) 64 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-07T03:16:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: audit 2024-03-07T03:15:59.528971+0000 mon.c (mon.2) 65 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: audit 2024-03-07T03:15:59.532359+0000 mon.c (mon.2) 66 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-07T03:16:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:00 smithi195 bash[23221]: audit 2024-03-07T03:15:59.533303+0000 mon.c (mon.2) 67 : audit 0 from='mgr.? 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:00.830 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:16:01.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:01 smithi148 bash[21362]: cluster 2024-03-07T03:16:00.447970+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2024-03-07T03:16:01.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:01 smithi148 bash[21362]: cluster 2024-03-07T03:16:00.449221+0000 mon.a (mon.0) 1118 : cluster 0 mgrmap e26: x(active, since 49s), standbys: y 2024-03-07T03:16:01.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:01 smithi148 bash[21362]: audit 2024-03-07T03:16:01.279955+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.148:0/1773369895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:01.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:01 smithi148 bash[31149]: cluster 2024-03-07T03:16:00.447970+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2024-03-07T03:16:01.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:01 smithi148 bash[31149]: cluster 2024-03-07T03:16:00.449221+0000 mon.a (mon.0) 1118 : cluster 0 mgrmap e26: x(active, since 49s), standbys: y 2024-03-07T03:16:01.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:01 smithi148 bash[31149]: audit 2024-03-07T03:16:01.279955+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.148:0/1773369895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:01 smithi195 bash[23221]: cluster 2024-03-07T03:16:00.447970+0000 mon.a (mon.0) 1117 : cluster 0 osdmap e219: 8 total, 8 up, 8 in 2024-03-07T03:16:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:01 smithi195 bash[23221]: cluster 2024-03-07T03:16:00.449221+0000 mon.a (mon.0) 1118 : cluster 0 mgrmap e26: x(active, since 49s), standbys: y 2024-03-07T03:16:01.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:01 smithi195 bash[23221]: audit 2024-03-07T03:16:01.279955+0000 mon.c (mon.2) 68 : audit 0 from='client.? 172.21.15.148:0/1773369895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:02.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:02 smithi148 bash[21362]: cluster 2024-03-07T03:16:01.056190+0000 mgr.x (mgr.24580) 49 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:02.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:02 smithi148 bash[21362]: cluster 2024-03-07T03:16:01.451840+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2024-03-07T03:16:02.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:02 smithi148 bash[31149]: cluster 2024-03-07T03:16:01.056190+0000 mgr.x (mgr.24580) 49 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:02.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:02 smithi148 bash[31149]: cluster 2024-03-07T03:16:01.451840+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2024-03-07T03:16:02.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:02 smithi195 bash[23221]: cluster 2024-03-07T03:16:01.056190+0000 mgr.x (mgr.24580) 49 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:02.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:02 smithi195 bash[23221]: cluster 2024-03-07T03:16:01.451840+0000 mon.a (mon.0) 1119 : cluster 0 osdmap e220: 8 total, 8 up, 8 in 2024-03-07T03:16:03.538 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:03 smithi148 bash[31149]: cluster 2024-03-07T03:16:02.460604+0000 mon.a (mon.0) 1120 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2024-03-07T03:16:03.538 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:03 smithi148 bash[21362]: cluster 2024-03-07T03:16:02.460604+0000 mon.a (mon.0) 1120 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2024-03-07T03:16:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:03 smithi195 bash[23221]: cluster 2024-03-07T03:16:02.460604+0000 mon.a (mon.0) 1120 : cluster 0 osdmap e221: 8 total, 8 up, 8 in 2024-03-07T03:16:03.813 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:03 smithi195 bash[57805]: ts=2024-03-07T03:16:03.511Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:03.890 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (15790 ms) 2024-03-07T03:16:03.890 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-03-07T03:16:03.972 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:03.998 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:16:04.114 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:16:04.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:04 smithi148 bash[21362]: cluster 2024-03-07T03:16:03.057025+0000 mgr.x (mgr.24580) 50 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 538 op/s 2024-03-07T03:16:04.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:04 smithi148 bash[21362]: cluster 2024-03-07T03:16:03.474476+0000 mon.a (mon.0) 1121 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2024-03-07T03:16:04.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:04 smithi148 bash[21362]: audit 2024-03-07T03:16:03.764953+0000 mon.a (mon.0) 1122 : audit 0 from='client.? 172.21.15.148:0/2221531157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:04.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:04 smithi148 bash[31149]: cluster 2024-03-07T03:16:03.057025+0000 mgr.x (mgr.24580) 50 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 538 op/s 2024-03-07T03:16:04.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:04 smithi148 bash[31149]: cluster 2024-03-07T03:16:03.474476+0000 mon.a (mon.0) 1121 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2024-03-07T03:16:04.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:04 smithi148 bash[31149]: audit 2024-03-07T03:16:03.764953+0000 mon.a (mon.0) 1122 : audit 0 from='client.? 172.21.15.148:0/2221531157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:04.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:04 smithi195 bash[23221]: cluster 2024-03-07T03:16:03.057025+0000 mgr.x (mgr.24580) 50 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 538 op/s 2024-03-07T03:16:04.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:04 smithi195 bash[23221]: cluster 2024-03-07T03:16:03.474476+0000 mon.a (mon.0) 1121 : cluster 0 osdmap e222: 8 total, 8 up, 8 in 2024-03-07T03:16:04.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:04 smithi195 bash[23221]: audit 2024-03-07T03:16:03.764953+0000 mon.a (mon.0) 1122 : audit 0 from='client.? 172.21.15.148:0/2221531157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:05.496 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:16:05.539 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:16:05.539 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:16:05.539 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:16:05.746 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:05 smithi148 bash[21362]: cluster 2024-03-07T03:16:04.477273+0000 mon.a (mon.0) 1123 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2024-03-07T03:16:05.747 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:05 smithi148 bash[31149]: cluster 2024-03-07T03:16:04.477273+0000 mon.a (mon.0) 1123 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2024-03-07T03:16:05.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:05 smithi195 bash[23221]: cluster 2024-03-07T03:16:04.477273+0000 mon.a (mon.0) 1123 : cluster 0 osdmap e223: 8 total, 8 up, 8 in 2024-03-07T03:16:06.677 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:16:06.683 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:16:06.683 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:16:06.695 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:16:06.718 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:16:06.727 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:06 smithi195 bash[23221]: cluster 2024-03-07T03:16:05.057827+0000 mgr.x (mgr.24580) 51 : cluster 0 pgmap v82: 137 pgs: 137 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 538 op/s 2024-03-07T03:16:06.727 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:06 smithi195 bash[23221]: cluster 2024-03-07T03:16:05.489019+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2024-03-07T03:16:06.727 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:06 smithi195 bash[23221]: audit 2024-03-07T03:16:06.201750+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.148:0/924333146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:06.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[21362]: cluster 2024-03-07T03:16:05.057827+0000 mgr.x (mgr.24580) 51 : cluster 0 pgmap v82: 137 pgs: 137 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 538 op/s 2024-03-07T03:16:06.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[21362]: cluster 2024-03-07T03:16:05.489019+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2024-03-07T03:16:06.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[21362]: audit 2024-03-07T03:16:06.201750+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.148:0/924333146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:06.788 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: [07/Mar/2024:03:16:06] ENGINE Bus STOPPING 2024-03-07T03:16:06.788 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: [07/Mar/2024:03:16:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:16:06.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: [07/Mar/2024:03:16:06] ENGINE Bus STOPPED 2024-03-07T03:16:06.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[31149]: cluster 2024-03-07T03:16:05.057827+0000 mgr.x (mgr.24580) 51 : cluster 0 pgmap v82: 137 pgs: 137 active+clean; 693 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 538 op/s 2024-03-07T03:16:06.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[31149]: cluster 2024-03-07T03:16:05.489019+0000 mon.a (mon.0) 1124 : cluster 0 osdmap e224: 8 total, 8 up, 8 in 2024-03-07T03:16:06.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[31149]: audit 2024-03-07T03:16:06.201750+0000 mon.c (mon.2) 69 : audit 0 from='client.? 172.21.15.148:0/924333146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:07.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:06 smithi195 bash[51559]: ignoring --setuser ceph since I am not root 2024-03-07T03:16:07.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:06 smithi195 bash[51559]: ignoring --setgroup ceph since I am not root 2024-03-07T03:16:07.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:06 smithi195 bash[51559]: debug 2024-03-07T03:16:06.736+0000 7fec97c81700 1 -- 172.21.15.195:0/1284654583 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561bf80465a0 con 0x561bf8dc6800 2024-03-07T03:16:07.063 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:06 smithi195 bash[51559]: debug 2024-03-07T03:16:06.888+0000 7feca1a65200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:16:07.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: [07/Mar/2024:03:16:06] ENGINE Bus STARTING 2024-03-07T03:16:07.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.880+0000 7fef02646700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.880+0000 7fef02646700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.880+0000 7fef02646700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.880+0000 7fef02646700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.288 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.880+0000 7fef02646700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.880+0000 7fef0564c700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.880+0000 7fef0564c700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.884+0000 7fef0564c700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.884+0000 7fef0564c700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: debug 2024-03-07T03:16:06.884+0000 7fef0564c700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:16:07.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: [07/Mar/2024:03:16:06] ENGINE Serving on http://:::9283 2024-03-07T03:16:07.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:06 smithi148 bash[72584]: [07/Mar/2024:03:16:06] ENGINE Bus STARTED 2024-03-07T03:16:07.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: cluster 2024-03-07T03:16:06.497270+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.511575+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.530410+0000 mon.a (mon.0) 1127 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.532712+0000 mon.b (mon.1) 188 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.534854+0000 mon.b (mon.1) 189 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.547385+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.643156+0000 mon.b (mon.1) 190 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.654533+0000 mon.b (mon.1) 191 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.657677+0000 mon.a (mon.0) 1129 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: cluster 2024-03-07T03:16:06.671066+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.672103+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-07T03:16:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: cluster 2024-03-07T03:16:06.672377+0000 mon.a (mon.0) 1132 : cluster 0 mgrmap e27: y(active, starting, since 0.0135238s) 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.677490+0000 mon.c (mon.2) 70 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.677763+0000 mon.c (mon.2) 71 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.679012+0000 mon.c (mon.2) 72 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.679365+0000 mon.c (mon.2) 73 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.679722+0000 mon.c (mon.2) 74 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.680758+0000 mon.c (mon.2) 75 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.681487+0000 mon.c (mon.2) 76 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.682103+0000 mon.c (mon.2) 77 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:16:07.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: cluster 2024-03-07T03:16:06.497270+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.511575+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.530410+0000 mon.a (mon.0) 1127 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.532712+0000 mon.b (mon.1) 188 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.534854+0000 mon.b (mon.1) 189 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.547385+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.643156+0000 mon.b (mon.1) 190 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.654533+0000 mon.b (mon.1) 191 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.657677+0000 mon.a (mon.0) 1129 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: cluster 2024-03-07T03:16:06.671066+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.672103+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-07T03:16:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: cluster 2024-03-07T03:16:06.672377+0000 mon.a (mon.0) 1132 : cluster 0 mgrmap e27: y(active, starting, since 0.0135238s) 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.677490+0000 mon.c (mon.2) 70 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.677763+0000 mon.c (mon.2) 71 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.679012+0000 mon.c (mon.2) 72 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.679365+0000 mon.c (mon.2) 73 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.679722+0000 mon.c (mon.2) 74 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.680758+0000 mon.c (mon.2) 75 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.681487+0000 mon.c (mon.2) 76 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:16:07.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.682103+0000 mon.c (mon.2) 77 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.682443+0000 mon.c (mon.2) 78 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.682752+0000 mon.c (mon.2) 79 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.683428+0000 mon.c (mon.2) 80 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.683787+0000 mon.c (mon.2) 81 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.684130+0000 mon.c (mon.2) 82 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.684408+0000 mon.c (mon.2) 83 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.684849+0000 mon.c (mon.2) 84 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.685253+0000 mon.c (mon.2) 85 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.686081+0000 mon.c (mon.2) 86 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: cluster 2024-03-07T03:16:06.738741+0000 mon.a (mon.0) 1133 : cluster 1 Manager daemon y is now available 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.780560+0000 mon.c (mon.2) 87 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.794 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.782817+0000 mon.c (mon.2) 88 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.813206+0000 mon.c (mon.2) 89 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.819254+0000 mon.c (mon.2) 90 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.819656+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.841051+0000 mon.c (mon.2) 91 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[21362]: audit 2024-03-07T03:16:06.841519+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.682443+0000 mon.c (mon.2) 78 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.682752+0000 mon.c (mon.2) 79 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.683428+0000 mon.c (mon.2) 80 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:16:07.795 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.683787+0000 mon.c (mon.2) 81 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.684130+0000 mon.c (mon.2) 82 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.684408+0000 mon.c (mon.2) 83 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.684849+0000 mon.c (mon.2) 84 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.685253+0000 mon.c (mon.2) 85 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.686081+0000 mon.c (mon.2) 86 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: cluster 2024-03-07T03:16:06.738741+0000 mon.a (mon.0) 1133 : cluster 1 Manager daemon y is now available 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.780560+0000 mon.c (mon.2) 87 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.782817+0000 mon.c (mon.2) 88 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.813206+0000 mon.c (mon.2) 89 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.819254+0000 mon.c (mon.2) 90 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.819656+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:16:07.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.841051+0000 mon.c (mon.2) 91 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:16:07.797 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:07 smithi148 bash[31149]: audit 2024-03-07T03:16:06.841519+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: cluster 2024-03-07T03:16:06.497270+0000 mon.a (mon.0) 1125 : cluster 0 osdmap e225: 8 total, 8 up, 8 in 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.511575+0000 mon.a (mon.0) 1126 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.530410+0000 mon.a (mon.0) 1127 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.532712+0000 mon.b (mon.1) 188 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.534854+0000 mon.b (mon.1) 189 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.547385+0000 mon.a (mon.0) 1128 : audit 1 from='mgr.24580 ' entity='mgr.x' 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.643156+0000 mon.b (mon.1) 190 : audit 0 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.654533+0000 mon.b (mon.1) 191 : audit 1 from='mgr.24580 172.21.15.195:0/335499174' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.657677+0000 mon.a (mon.0) 1129 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: cluster 2024-03-07T03:16:06.671066+0000 mon.a (mon.0) 1130 : cluster 0 osdmap e226: 8 total, 8 up, 8 in 2024-03-07T03:16:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.672103+0000 mon.a (mon.0) 1131 : audit 1 from='mgr.24580 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: cluster 2024-03-07T03:16:06.672377+0000 mon.a (mon.0) 1132 : cluster 0 mgrmap e27: y(active, starting, since 0.0135238s) 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.677490+0000 mon.c (mon.2) 70 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.677763+0000 mon.c (mon.2) 71 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.679012+0000 mon.c (mon.2) 72 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.679365+0000 mon.c (mon.2) 73 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.679722+0000 mon.c (mon.2) 74 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.680758+0000 mon.c (mon.2) 75 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.681487+0000 mon.c (mon.2) 76 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:16:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.682103+0000 mon.c (mon.2) 77 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.682443+0000 mon.c (mon.2) 78 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.682752+0000 mon.c (mon.2) 79 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.683428+0000 mon.c (mon.2) 80 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.683787+0000 mon.c (mon.2) 81 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.684130+0000 mon.c (mon.2) 82 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.684408+0000 mon.c (mon.2) 83 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.684849+0000 mon.c (mon.2) 84 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.685253+0000 mon.c (mon.2) 85 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.686081+0000 mon.c (mon.2) 86 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:16:07.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: cluster 2024-03-07T03:16:06.738741+0000 mon.a (mon.0) 1133 : cluster 1 Manager daemon y is now available 2024-03-07T03:16:07.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.780560+0000 mon.c (mon.2) 87 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.782817+0000 mon.c (mon.2) 88 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:07.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.813206+0000 mon.c (mon.2) 89 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:07.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.819254+0000 mon.c (mon.2) 90 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:16:07.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.819656+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:16:07.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.841051+0000 mon.c (mon.2) 91 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:16:07.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:07 smithi195 bash[23221]: audit 2024-03-07T03:16:06.841519+0000 mon.a (mon.0) 1135 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:16:08.345 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:08.374 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:16:08.386 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:08.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[51559]: debug 2024-03-07T03:16:08.520+0000 7feca1a65200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:16:08.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[51559]: debug 2024-03-07T03:16:08.624+0000 7feca1a65200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:16:08.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[23221]: cluster 2024-03-07T03:16:07.666480+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2024-03-07T03:16:08.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[23221]: cluster 2024-03-07T03:16:07.681701+0000 mon.a (mon.0) 1137 : cluster 0 mgrmap e28: y(active, since 1.02286s) 2024-03-07T03:16:08.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[23221]: cluster 2024-03-07T03:16:07.725063+0000 mgr.y (mgr.25052) 1 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:08.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[23221]: cluster 2024-03-07T03:16:08.345204+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2024-03-07T03:16:08.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[23221]: audit 2024-03-07T03:16:08.653131+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.148:0/2867793714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:09.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[21362]: cluster 2024-03-07T03:16:07.666480+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[21362]: cluster 2024-03-07T03:16:07.681701+0000 mon.a (mon.0) 1137 : cluster 0 mgrmap e28: y(active, since 1.02286s) 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[21362]: cluster 2024-03-07T03:16:07.725063+0000 mgr.y (mgr.25052) 1 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[21362]: cluster 2024-03-07T03:16:08.345204+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[21362]: audit 2024-03-07T03:16:08.653131+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.148:0/2867793714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[31149]: cluster 2024-03-07T03:16:07.666480+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e227: 8 total, 8 up, 8 in 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[31149]: cluster 2024-03-07T03:16:07.681701+0000 mon.a (mon.0) 1137 : cluster 0 mgrmap e28: y(active, since 1.02286s) 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[31149]: cluster 2024-03-07T03:16:07.725063+0000 mgr.y (mgr.25052) 1 : cluster 0 pgmap v4: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[31149]: cluster 2024-03-07T03:16:08.345204+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e228: 8 total, 8 up, 8 in 2024-03-07T03:16:09.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:08 smithi148 bash[31149]: audit 2024-03-07T03:16:08.653131+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.148:0/2867793714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:09.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:08 smithi195 bash[51559]: debug 2024-03-07T03:16:08.900+0000 7feca1a65200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[51559]: debug 2024-03-07T03:16:09.716+0000 7feca1a65200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[51559]: debug 2024-03-07T03:16:09.828+0000 7feca1a65200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[23221]: cephadm 2024-03-07T03:16:07.938599+0000 mgr.y (mgr.25052) 2 : cephadm 1 [07/Mar/2024:03:16:07] ENGINE Bus STARTING 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[23221]: cephadm 2024-03-07T03:16:08.040620+0000 mgr.y (mgr.25052) 3 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[23221]: cephadm 2024-03-07T03:16:08.158194+0000 mgr.y (mgr.25052) 4 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[23221]: cephadm 2024-03-07T03:16:08.158395+0000 mgr.y (mgr.25052) 5 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Bus STARTED 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[23221]: cluster 2024-03-07T03:16:08.683306+0000 mgr.y (mgr.25052) 6 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:09.924 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[23221]: cluster 2024-03-07T03:16:09.357413+0000 mon.a (mon.0) 1139 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2024-03-07T03:16:10.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[31149]: cephadm 2024-03-07T03:16:07.938599+0000 mgr.y (mgr.25052) 2 : cephadm 1 [07/Mar/2024:03:16:07] ENGINE Bus STARTING 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[31149]: cephadm 2024-03-07T03:16:08.040620+0000 mgr.y (mgr.25052) 3 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[31149]: cephadm 2024-03-07T03:16:08.158194+0000 mgr.y (mgr.25052) 4 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[31149]: cephadm 2024-03-07T03:16:08.158395+0000 mgr.y (mgr.25052) 5 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Bus STARTED 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[31149]: cluster 2024-03-07T03:16:08.683306+0000 mgr.y (mgr.25052) 6 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[31149]: cluster 2024-03-07T03:16:09.357413+0000 mon.a (mon.0) 1139 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[21362]: cephadm 2024-03-07T03:16:07.938599+0000 mgr.y (mgr.25052) 2 : cephadm 1 [07/Mar/2024:03:16:07] ENGINE Bus STARTING 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[21362]: cephadm 2024-03-07T03:16:08.040620+0000 mgr.y (mgr.25052) 3 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[21362]: cephadm 2024-03-07T03:16:08.158194+0000 mgr.y (mgr.25052) 4 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[21362]: cephadm 2024-03-07T03:16:08.158395+0000 mgr.y (mgr.25052) 5 : cephadm 1 [07/Mar/2024:03:16:08] ENGINE Bus STARTED 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[21362]: cluster 2024-03-07T03:16:08.683306+0000 mgr.y (mgr.25052) 6 : cluster 0 pgmap v6: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:10.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:09 smithi148 bash[21362]: cluster 2024-03-07T03:16:09.357413+0000 mon.a (mon.0) 1139 : cluster 0 osdmap e229: 8 total, 8 up, 8 in 2024-03-07T03:16:10.284 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:09 smithi195 bash[51559]: debug 2024-03-07T03:16:09.932+0000 7feca1a65200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:16:10.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:10 smithi195 bash[51559]: debug 2024-03-07T03:16:10.292+0000 7feca1a65200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:16:10.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:10 smithi195 bash[51559]: debug 2024-03-07T03:16:10.412+0000 7feca1a65200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:16:10.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:10 smithi195 bash[51559]: debug 2024-03-07T03:16:10.516+0000 7feca1a65200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:16:11.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:10 smithi195 bash[51559]: debug 2024-03-07T03:16:10.880+0000 7feca1a65200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:16:11.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[51559]: debug 2024-03-07T03:16:11.120+0000 7feca1a65200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:16:11.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[21362]: cluster 2024-03-07T03:16:10.360136+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2024-03-07T03:16:11.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[21362]: cluster 2024-03-07T03:16:10.380657+0000 mon.a (mon.0) 1141 : cluster 0 mgrmap e29: y(active, since 3s) 2024-03-07T03:16:11.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[21362]: cluster 2024-03-07T03:16:10.684199+0000 mgr.y (mgr.25052) 7 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:11.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[21362]: audit 2024-03-07T03:16:11.090580+0000 mon.a (mon.0) 1142 : audit 0 from='client.? 172.21.15.148:0/3137469680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[31149]: cluster 2024-03-07T03:16:10.360136+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2024-03-07T03:16:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[31149]: cluster 2024-03-07T03:16:10.380657+0000 mon.a (mon.0) 1141 : cluster 0 mgrmap e29: y(active, since 3s) 2024-03-07T03:16:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[31149]: cluster 2024-03-07T03:16:10.684199+0000 mgr.y (mgr.25052) 7 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:11 smithi148 bash[31149]: audit 2024-03-07T03:16:11.090580+0000 mon.a (mon.0) 1142 : audit 0 from='client.? 172.21.15.148:0/3137469680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[23221]: cluster 2024-03-07T03:16:10.360136+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e230: 8 total, 8 up, 8 in 2024-03-07T03:16:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[23221]: cluster 2024-03-07T03:16:10.380657+0000 mon.a (mon.0) 1141 : cluster 0 mgrmap e29: y(active, since 3s) 2024-03-07T03:16:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[23221]: cluster 2024-03-07T03:16:10.684199+0000 mgr.y (mgr.25052) 7 : cluster 0 pgmap v9: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[23221]: audit 2024-03-07T03:16:11.090580+0000 mon.a (mon.0) 1142 : audit 0 from='client.? 172.21.15.148:0/3137469680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:11.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[51559]: debug 2024-03-07T03:16:11.388+0000 7feca1a65200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:16:11.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[51559]: debug 2024-03-07T03:16:11.520+0000 7feca1a65200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:16:12.069 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:11 smithi195 bash[51559]: debug 2024-03-07T03:16:11.820+0000 7feca1a65200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:16:12.195 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: after_prepare_snap 2024-03-07T03:16:12.371 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:12 smithi195 bash[51559]: debug 2024-03-07T03:16:12.076+0000 7feca1a65200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:16:12.651 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:12 smithi195 bash[23221]: cluster 2024-03-07T03:16:11.384692+0000 mon.a (mon.0) 1143 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2024-03-07T03:16:12.652 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:12 smithi195 bash[51559]: debug 2024-03-07T03:16:12.528+0000 7feca1a65200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:16:12.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:12 smithi148 bash[21362]: cluster 2024-03-07T03:16:11.384692+0000 mon.a (mon.0) 1143 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2024-03-07T03:16:12.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:12 smithi148 bash[31149]: cluster 2024-03-07T03:16:11.384692+0000 mon.a (mon.0) 1143 : cluster 0 osdmap e231: 8 total, 8 up, 8 in 2024-03-07T03:16:12.926 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:12 smithi195 bash[51559]: debug 2024-03-07T03:16:12.660+0000 7feca1a65200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:16:13.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:12 smithi195 bash[51559]: debug 2024-03-07T03:16:12.932+0000 7feca1a65200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:16:13.357 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:16:13.372 INFO:tasks.workunit.client.0.smithi148.stdout:write: * 83886080~1000 2024-03-07T03:16:13.383 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:16:13.581 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:16:13.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[21362]: cluster 2024-03-07T03:16:12.384754+0000 mon.a (mon.0) 1144 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2024-03-07T03:16:13.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[21362]: cluster 2024-03-07T03:16:12.684670+0000 mgr.y (mgr.25052) 8 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:13.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[21362]: audit 2024-03-07T03:16:12.961760+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[21362]: audit 2024-03-07T03:16:12.975499+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[21362]: audit 2024-03-07T03:16:13.167407+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[21362]: audit 2024-03-07T03:16:13.179976+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[21362]: cluster 2024-03-07T03:16:13.346833+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[31149]: cluster 2024-03-07T03:16:12.384754+0000 mon.a (mon.0) 1144 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[31149]: cluster 2024-03-07T03:16:12.684670+0000 mgr.y (mgr.25052) 8 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[31149]: audit 2024-03-07T03:16:12.961760+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[31149]: audit 2024-03-07T03:16:12.975499+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[31149]: audit 2024-03-07T03:16:13.167407+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[31149]: audit 2024-03-07T03:16:13.179976+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:13 smithi148 bash[31149]: cluster 2024-03-07T03:16:13.346833+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2024-03-07T03:16:13.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[57805]: ts=2024-03-07T03:16:13.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[23221]: cluster 2024-03-07T03:16:12.384754+0000 mon.a (mon.0) 1144 : cluster 0 osdmap e232: 8 total, 8 up, 8 in 2024-03-07T03:16:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[23221]: cluster 2024-03-07T03:16:12.684670+0000 mgr.y (mgr.25052) 8 : cluster 0 pgmap v12: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:16:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[23221]: audit 2024-03-07T03:16:12.961760+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[23221]: audit 2024-03-07T03:16:12.975499+0000 mon.a (mon.0) 1146 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[23221]: audit 2024-03-07T03:16:13.167407+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[23221]: audit 2024-03-07T03:16:13.179976+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:13.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:13 smithi195 bash[23221]: cluster 2024-03-07T03:16:13.346833+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e233: 8 total, 8 up, 8 in 2024-03-07T03:16:14.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[51559]: debug 2024-03-07T03:16:14.048+0000 7feca1a65200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:16:14.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[51559]: debug 2024-03-07T03:16:14.172+0000 7feca1a65200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:16:14.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[21362]: audit 2024-03-07T03:16:13.593248+0000 mon.a (mon.0) 1150 : audit 0 from='client.? 172.21.15.148:0/4244503349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[21362]: audit 2024-03-07T03:16:14.042140+0000 mon.a (mon.0) 1151 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[21362]: audit 2024-03-07T03:16:14.056810+0000 mon.a (mon.0) 1152 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[21362]: audit 2024-03-07T03:16:14.285062+0000 mon.a (mon.0) 1153 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[21362]: audit 2024-03-07T03:16:14.300325+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[31149]: audit 2024-03-07T03:16:13.593248+0000 mon.a (mon.0) 1150 : audit 0 from='client.? 172.21.15.148:0/4244503349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[31149]: audit 2024-03-07T03:16:14.042140+0000 mon.a (mon.0) 1151 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[31149]: audit 2024-03-07T03:16:14.056810+0000 mon.a (mon.0) 1152 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[31149]: audit 2024-03-07T03:16:14.285062+0000 mon.a (mon.0) 1153 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:14 smithi148 bash[31149]: audit 2024-03-07T03:16:14.300325+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[23221]: audit 2024-03-07T03:16:13.593248+0000 mon.a (mon.0) 1150 : audit 0 from='client.? 172.21.15.148:0/4244503349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:14.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[23221]: audit 2024-03-07T03:16:14.042140+0000 mon.a (mon.0) 1151 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[23221]: audit 2024-03-07T03:16:14.056810+0000 mon.a (mon.0) 1152 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[23221]: audit 2024-03-07T03:16:14.285062+0000 mon.a (mon.0) 1153 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[23221]: audit 2024-03-07T03:16:14.300325+0000 mon.a (mon.0) 1154 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:14.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[51559]: debug 2024-03-07T03:16:14.484+0000 7feca1a65200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:16:14.813 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:14 smithi195 bash[51559]: debug 2024-03-07T03:16:14.600+0000 7feca1a65200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:16:15.311 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[51559]: debug 2024-03-07T03:16:15.064+0000 7feca1a65200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:16:15.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[51559]: debug 2024-03-07T03:16:15.188+0000 7feca1a65200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:16:15.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[51559]: [07/Mar/2024:03:16:15] ENGINE Bus STARTING 2024-03-07T03:16:15.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[51559]: CherryPy Checker: 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[51559]: The Application mounted at '' has an empty config. 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[51559]: [07/Mar/2024:03:16:15] ENGINE Serving on http://:::9283 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[51559]: [07/Mar/2024:03:16:15] ENGINE Bus STARTED 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[23221]: cluster 2024-03-07T03:16:14.408628+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[23221]: cluster 2024-03-07T03:16:14.685403+0000 mgr.y (mgr.25052) 9 : cluster 0 pgmap v15: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 308 op/s 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[23221]: cluster 2024-03-07T03:16:15.218792+0000 mon.a (mon.0) 1156 : cluster 0 Standby manager daemon x started 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[23221]: audit 2024-03-07T03:16:15.220541+0000 mon.b (mon.1) 192 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[23221]: audit 2024-03-07T03:16:15.221561+0000 mon.b (mon.1) 193 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[23221]: audit 2024-03-07T03:16:15.224324+0000 mon.b (mon.1) 194 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:16:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:15 smithi195 bash[23221]: audit 2024-03-07T03:16:15.224840+0000 mon.b (mon.1) 195 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[21362]: cluster 2024-03-07T03:16:14.408628+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2024-03-07T03:16:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[21362]: cluster 2024-03-07T03:16:14.685403+0000 mgr.y (mgr.25052) 9 : cluster 0 pgmap v15: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 308 op/s 2024-03-07T03:16:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[21362]: cluster 2024-03-07T03:16:15.218792+0000 mon.a (mon.0) 1156 : cluster 0 Standby manager daemon x started 2024-03-07T03:16:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[21362]: audit 2024-03-07T03:16:15.220541+0000 mon.b (mon.1) 192 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:16:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[21362]: audit 2024-03-07T03:16:15.221561+0000 mon.b (mon.1) 193 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[21362]: audit 2024-03-07T03:16:15.224324+0000 mon.b (mon.1) 194 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:16:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[21362]: audit 2024-03-07T03:16:15.224840+0000 mon.b (mon.1) 195 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[31149]: cluster 2024-03-07T03:16:14.408628+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e234: 8 total, 8 up, 8 in 2024-03-07T03:16:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[31149]: cluster 2024-03-07T03:16:14.685403+0000 mgr.y (mgr.25052) 9 : cluster 0 pgmap v15: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 308 op/s 2024-03-07T03:16:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[31149]: cluster 2024-03-07T03:16:15.218792+0000 mon.a (mon.0) 1156 : cluster 0 Standby manager daemon x started 2024-03-07T03:16:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[31149]: audit 2024-03-07T03:16:15.220541+0000 mon.b (mon.1) 192 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:16:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[31149]: audit 2024-03-07T03:16:15.221561+0000 mon.b (mon.1) 193 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[31149]: audit 2024-03-07T03:16:15.224324+0000 mon.b (mon.1) 194 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:16:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:15 smithi148 bash[31149]: audit 2024-03-07T03:16:15.224840+0000 mon.b (mon.1) 195 : audit 0 from='mgr.? 172.21.15.195:0/3834079756' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:16.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[21362]: cluster 2024-03-07T03:16:15.434150+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2024-03-07T03:16:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[21362]: cluster 2024-03-07T03:16:15.442680+0000 mon.a (mon.0) 1158 : cluster 0 mgrmap e30: y(active, since 8s), standbys: x 2024-03-07T03:16:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[21362]: audit 2024-03-07T03:16:15.444763+0000 mon.c (mon.2) 93 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:16:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[21362]: audit 2024-03-07T03:16:16.032730+0000 mon.a (mon.0) 1159 : audit 0 from='client.? 172.21.15.148:0/199321184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[31149]: cluster 2024-03-07T03:16:15.434150+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2024-03-07T03:16:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[31149]: cluster 2024-03-07T03:16:15.442680+0000 mon.a (mon.0) 1158 : cluster 0 mgrmap e30: y(active, since 8s), standbys: x 2024-03-07T03:16:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[31149]: audit 2024-03-07T03:16:15.444763+0000 mon.c (mon.2) 93 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:16:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:16 smithi148 bash[31149]: audit 2024-03-07T03:16:16.032730+0000 mon.a (mon.0) 1159 : audit 0 from='client.? 172.21.15.148:0/199321184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:16.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:16 smithi195 bash[23221]: cluster 2024-03-07T03:16:15.434150+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e235: 8 total, 8 up, 8 in 2024-03-07T03:16:16.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:16 smithi195 bash[23221]: cluster 2024-03-07T03:16:15.442680+0000 mon.a (mon.0) 1158 : cluster 0 mgrmap e30: y(active, since 8s), standbys: x 2024-03-07T03:16:16.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:16 smithi195 bash[23221]: audit 2024-03-07T03:16:15.444763+0000 mon.c (mon.2) 93 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:16:16.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:16 smithi195 bash[23221]: audit 2024-03-07T03:16:16.032730+0000 mon.a (mon.0) 1159 : audit 0 from='client.? 172.21.15.148:0/199321184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:17.128 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (13237 ms) 2024-03-07T03:16:17.128 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.Clone 2024-03-07T03:16:17.199 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:17.222 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~8388609 2024-03-07T03:16:17.338 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 16777218~8388609 2024-03-07T03:16:17.413 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap 2024-03-07T03:16:17.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:17 smithi148 bash[21362]: cluster 2024-03-07T03:16:16.434768+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2024-03-07T03:16:17.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:17 smithi148 bash[21362]: cluster 2024-03-07T03:16:16.686242+0000 mgr.y (mgr.25052) 10 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 308 op/s 2024-03-07T03:16:17.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:17 smithi148 bash[31149]: cluster 2024-03-07T03:16:16.434768+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2024-03-07T03:16:17.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:17 smithi148 bash[31149]: cluster 2024-03-07T03:16:16.686242+0000 mgr.y (mgr.25052) 10 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 308 op/s 2024-03-07T03:16:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:17 smithi195 bash[23221]: cluster 2024-03-07T03:16:16.434768+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e236: 8 total, 8 up, 8 in 2024-03-07T03:16:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:17 smithi195 bash[23221]: cluster 2024-03-07T03:16:16.686242+0000 mgr.y (mgr.25052) 10 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 32 MiB/s rd, 12 KiB/s wr, 308 op/s 2024-03-07T03:16:19.460 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap 2024-03-07T03:16:19.493 INFO:tasks.workunit.client.0.smithi148.stdout:clone image105 -> image108 2024-03-07T03:16:19.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:19 smithi195 bash[51559]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:16:19] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:16:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:19 smithi195 bash[23221]: cluster 2024-03-07T03:16:18.439324+0000 mon.a (mon.0) 1161 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2024-03-07T03:16:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:19 smithi195 bash[23221]: audit 2024-03-07T03:16:18.485044+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.148:0/3661842328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:19 smithi195 bash[23221]: cluster 2024-03-07T03:16:18.687033+0000 mgr.y (mgr.25052) 11 : cluster 0 pgmap v20: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 402 op/s 2024-03-07T03:16:19.746 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:16:19.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:19 smithi148 bash[21362]: cluster 2024-03-07T03:16:18.439324+0000 mon.a (mon.0) 1161 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2024-03-07T03:16:19.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:19 smithi148 bash[21362]: audit 2024-03-07T03:16:18.485044+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.148:0/3661842328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:19.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:19 smithi148 bash[21362]: cluster 2024-03-07T03:16:18.687033+0000 mgr.y (mgr.25052) 11 : cluster 0 pgmap v20: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 402 op/s 2024-03-07T03:16:19.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:19 smithi148 bash[31149]: cluster 2024-03-07T03:16:18.439324+0000 mon.a (mon.0) 1161 : cluster 0 osdmap e237: 8 total, 8 up, 8 in 2024-03-07T03:16:19.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:19 smithi148 bash[31149]: audit 2024-03-07T03:16:18.485044+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.148:0/3661842328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:19.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:19 smithi148 bash[31149]: cluster 2024-03-07T03:16:18.687033+0000 mgr.y (mgr.25052) 11 : cluster 0 pgmap v20: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 402 op/s 2024-03-07T03:16:19.840 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:16:19.845 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap 2024-03-07T03:16:20.453 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:20 smithi148 bash[21362]: cluster 2024-03-07T03:16:19.451886+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2024-03-07T03:16:20.453 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:20 smithi148 bash[31149]: cluster 2024-03-07T03:16:19.451886+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2024-03-07T03:16:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:20 smithi195 bash[23221]: cluster 2024-03-07T03:16:19.451886+0000 mon.a (mon.0) 1162 : cluster 0 osdmap e238: 8 total, 8 up, 8 in 2024-03-07T03:16:21.476 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap 2024-03-07T03:16:21.503 INFO:tasks.workunit.client.0.smithi148.stdout:clone image108 -> image110 2024-03-07T03:16:21.702 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:21 smithi148 bash[21362]: cluster 2024-03-07T03:16:20.467343+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2024-03-07T03:16:21.702 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:21 smithi148 bash[21362]: cluster 2024-03-07T03:16:20.687635+0000 mgr.y (mgr.25052) 12 : cluster 0 pgmap v23: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 404 op/s 2024-03-07T03:16:21.702 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:21 smithi148 bash[21362]: audit 2024-03-07T03:16:20.924908+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.148:0/433020936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:21.702 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:21 smithi148 bash[31149]: cluster 2024-03-07T03:16:20.467343+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2024-03-07T03:16:21.702 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:21 smithi148 bash[31149]: cluster 2024-03-07T03:16:20.687635+0000 mgr.y (mgr.25052) 12 : cluster 0 pgmap v23: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 404 op/s 2024-03-07T03:16:21.702 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:21 smithi148 bash[31149]: audit 2024-03-07T03:16:20.924908+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.148:0/433020936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:21.767 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 1000~1000 2024-03-07T03:16:21.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:21 smithi195 bash[23221]: cluster 2024-03-07T03:16:20.467343+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e239: 8 total, 8 up, 8 in 2024-03-07T03:16:21.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:21 smithi195 bash[23221]: cluster 2024-03-07T03:16:20.687635+0000 mgr.y (mgr.25052) 12 : cluster 0 pgmap v23: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 43 MiB/s rd, 13 MiB/s wr, 404 op/s 2024-03-07T03:16:21.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:21 smithi195 bash[23221]: audit 2024-03-07T03:16:20.924908+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.148:0/433020936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:21.839 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 2010~1000 2024-03-07T03:16:21.844 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:22.297 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:16:22.552 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:16:22.726 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:22 smithi148 bash[21362]: cluster 2024-03-07T03:16:21.471101+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2024-03-07T03:16:22.726 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:22 smithi148 bash[21362]: audit 2024-03-07T03:16:21.812011+0000 mon.c (mon.2) 96 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:22.726 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:22 smithi148 bash[31149]: cluster 2024-03-07T03:16:21.471101+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2024-03-07T03:16:22.726 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:22 smithi148 bash[31149]: audit 2024-03-07T03:16:21.812011+0000 mon.c (mon.2) 96 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:22.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:22 smithi195 bash[23221]: cluster 2024-03-07T03:16:21.471101+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e240: 8 total, 8 up, 8 in 2024-03-07T03:16:22.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:22 smithi195 bash[23221]: audit 2024-03-07T03:16:21.812011+0000 mon.c (mon.2) 96 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:22.816 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.Clone (5689 ms) 2024-03-07T03:16:22.816 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneParent 2024-03-07T03:16:22.883 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:22.907 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap 2024-03-07T03:16:23.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:23 smithi148 bash[21362]: cluster 2024-03-07T03:16:22.688116+0000 mgr.y (mgr.25052) 13 : cluster 0 pgmap v25: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 963 KiB/s rd, 0 op/s 2024-03-07T03:16:23.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:23 smithi148 bash[21362]: cluster 2024-03-07T03:16:23.346536+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2024-03-07T03:16:23.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:23 smithi148 bash[21362]: audit 2024-03-07T03:16:23.366281+0000 mon.a (mon.0) 1166 : audit 0 from='client.? 172.21.15.148:0/577671784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:23 smithi148 bash[31149]: cluster 2024-03-07T03:16:22.688116+0000 mgr.y (mgr.25052) 13 : cluster 0 pgmap v25: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 963 KiB/s rd, 0 op/s 2024-03-07T03:16:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:23 smithi148 bash[31149]: cluster 2024-03-07T03:16:23.346536+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2024-03-07T03:16:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:23 smithi148 bash[31149]: audit 2024-03-07T03:16:23.366281+0000 mon.a (mon.0) 1166 : audit 0 from='client.? 172.21.15.148:0/577671784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:23.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:23 smithi195 bash[57805]: ts=2024-03-07T03:16:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:23.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:23 smithi195 bash[23221]: cluster 2024-03-07T03:16:22.688116+0000 mgr.y (mgr.25052) 13 : cluster 0 pgmap v25: 137 pgs: 137 active+clean; 761 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 963 KiB/s rd, 0 op/s 2024-03-07T03:16:23.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:23 smithi195 bash[23221]: cluster 2024-03-07T03:16:23.346536+0000 mon.a (mon.0) 1165 : cluster 0 osdmap e241: 8 total, 8 up, 8 in 2024-03-07T03:16:23.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:23 smithi195 bash[23221]: audit 2024-03-07T03:16:23.366281+0000 mon.a (mon.0) 1166 : audit 0 from='client.? 172.21.15.148:0/577671784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:24.372 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap 2024-03-07T03:16:24.396 INFO:tasks.workunit.client.0.smithi148.stdout:clone image111 -> image114 2024-03-07T03:16:24.594 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:23.486875+0000 mon.a (mon.0) 1167 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:23.500919+0000 mon.a (mon.0) 1168 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:23.506883+0000 mon.c (mon.2) 97 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:23.508656+0000 mon.c (mon.2) 98 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:24.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: cephadm 2024-03-07T03:16:23.511259+0000 mgr.y (mgr.25052) 14 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: cephadm 2024-03-07T03:16:23.511635+0000 mgr.y (mgr.25052) 15 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: cephadm 2024-03-07T03:16:23.776531+0000 mgr.y (mgr.25052) 16 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: cephadm 2024-03-07T03:16:23.777738+0000 mgr.y (mgr.25052) 17 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:24.276892+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:24.282887+0000 mon.a (mon.0) 1170 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:24.295323+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:24.302171+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: audit 2024-03-07T03:16:24.312968+0000 mon.a (mon.0) 1173 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[31149]: cluster 2024-03-07T03:16:24.356400+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2024-03-07T03:16:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:23.486875+0000 mon.a (mon.0) 1167 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:23.500919+0000 mon.a (mon.0) 1168 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:23.506883+0000 mon.c (mon.2) 97 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:23.508656+0000 mon.c (mon.2) 98 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: cephadm 2024-03-07T03:16:23.511259+0000 mgr.y (mgr.25052) 14 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: cephadm 2024-03-07T03:16:23.511635+0000 mgr.y (mgr.25052) 15 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: cephadm 2024-03-07T03:16:23.776531+0000 mgr.y (mgr.25052) 16 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: cephadm 2024-03-07T03:16:23.777738+0000 mgr.y (mgr.25052) 17 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:24.276892+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:24.282887+0000 mon.a (mon.0) 1170 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:24.295323+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:24.302171+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: audit 2024-03-07T03:16:24.312968+0000 mon.a (mon.0) 1173 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:24 smithi148 bash[21362]: cluster 2024-03-07T03:16:24.356400+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2024-03-07T03:16:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:23.486875+0000 mon.a (mon.0) 1167 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:23.500919+0000 mon.a (mon.0) 1168 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:23.506883+0000 mon.c (mon.2) 97 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:23.508656+0000 mon.c (mon.2) 98 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: cephadm 2024-03-07T03:16:23.511259+0000 mgr.y (mgr.25052) 14 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: cephadm 2024-03-07T03:16:23.511635+0000 mgr.y (mgr.25052) 15 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: cephadm 2024-03-07T03:16:23.776531+0000 mgr.y (mgr.25052) 16 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: cephadm 2024-03-07T03:16:23.777738+0000 mgr.y (mgr.25052) 17 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:24.276892+0000 mon.a (mon.0) 1169 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:24.282887+0000 mon.a (mon.0) 1170 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:24.295323+0000 mon.a (mon.0) 1171 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:24.302171+0000 mon.a (mon.0) 1172 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: audit 2024-03-07T03:16:24.312968+0000 mon.a (mon.0) 1173 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:24 smithi195 bash[23221]: cluster 2024-03-07T03:16:24.356400+0000 mon.a (mon.0) 1174 : cluster 0 osdmap e242: 8 total, 8 up, 8 in 2024-03-07T03:16:24.864 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:16:25.139 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneParent (2322 ms) 2024-03-07T03:16:25.139 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-03-07T03:16:25.212 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~10 2024-03-07T03:16:25.260 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap 2024-03-07T03:16:25.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[21362]: cephadm 2024-03-07T03:16:23.941160+0000 mgr.y (mgr.25052) 18 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[21362]: cephadm 2024-03-07T03:16:23.942414+0000 mgr.y (mgr.25052) 19 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[21362]: cephadm 2024-03-07T03:16:24.105324+0000 mgr.y (mgr.25052) 20 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[21362]: cephadm 2024-03-07T03:16:24.107069+0000 mgr.y (mgr.25052) 21 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[21362]: cephadm 2024-03-07T03:16:24.392887+0000 mgr.y (mgr.25052) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[21362]: cephadm 2024-03-07T03:16:24.656271+0000 mgr.y (mgr.25052) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[21362]: cluster 2024-03-07T03:16:24.688561+0000 mgr.y (mgr.25052) 24 : cluster 0 pgmap v28: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.7 MiB/s wr, 727 op/s 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[31149]: cephadm 2024-03-07T03:16:23.941160+0000 mgr.y (mgr.25052) 18 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[31149]: cephadm 2024-03-07T03:16:23.942414+0000 mgr.y (mgr.25052) 19 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[31149]: cephadm 2024-03-07T03:16:24.105324+0000 mgr.y (mgr.25052) 20 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[31149]: cephadm 2024-03-07T03:16:24.107069+0000 mgr.y (mgr.25052) 21 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:16:25.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[31149]: cephadm 2024-03-07T03:16:24.392887+0000 mgr.y (mgr.25052) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:16:25.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[31149]: cephadm 2024-03-07T03:16:24.656271+0000 mgr.y (mgr.25052) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:16:25.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:25 smithi148 bash[31149]: cluster 2024-03-07T03:16:24.688561+0000 mgr.y (mgr.25052) 24 : cluster 0 pgmap v28: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.7 MiB/s wr, 727 op/s 2024-03-07T03:16:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:25 smithi195 bash[23221]: cephadm 2024-03-07T03:16:23.941160+0000 mgr.y (mgr.25052) 18 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:16:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:25 smithi195 bash[23221]: cephadm 2024-03-07T03:16:23.942414+0000 mgr.y (mgr.25052) 19 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:16:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:25 smithi195 bash[23221]: cephadm 2024-03-07T03:16:24.105324+0000 mgr.y (mgr.25052) 20 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:16:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:25 smithi195 bash[23221]: cephadm 2024-03-07T03:16:24.107069+0000 mgr.y (mgr.25052) 21 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:16:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:25 smithi195 bash[23221]: cephadm 2024-03-07T03:16:24.392887+0000 mgr.y (mgr.25052) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-03-07T03:16:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:25 smithi195 bash[23221]: cephadm 2024-03-07T03:16:24.656271+0000 mgr.y (mgr.25052) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi195 2024-03-07T03:16:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:25 smithi195 bash[23221]: cluster 2024-03-07T03:16:24.688561+0000 mgr.y (mgr.25052) 24 : cluster 0 pgmap v28: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 4.7 MiB/s wr, 727 op/s 2024-03-07T03:16:26.520 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap 2024-03-07T03:16:26.547 INFO:tasks.workunit.client.0.smithi148.stdout:clone image115 -> image118 2024-03-07T03:16:26.767 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:26 smithi195 bash[23221]: cluster 2024-03-07T03:16:25.500357+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2024-03-07T03:16:26.767 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:26 smithi195 bash[23221]: audit 2024-03-07T03:16:25.815096+0000 mon.a (mon.0) 1176 : audit 0 from='client.? 172.21.15.148:0/2226979094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:26.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:26 smithi148 bash[21362]: cluster 2024-03-07T03:16:25.500357+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2024-03-07T03:16:26.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:26 smithi148 bash[21362]: audit 2024-03-07T03:16:25.815096+0000 mon.a (mon.0) 1176 : audit 0 from='client.? 172.21.15.148:0/2226979094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:26.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:26 smithi148 bash[31149]: cluster 2024-03-07T03:16:25.500357+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e243: 8 total, 8 up, 8 in 2024-03-07T03:16:26.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:26 smithi148 bash[31149]: audit 2024-03-07T03:16:25.815096+0000 mon.a (mon.0) 1176 : audit 0 from='client.? 172.21.15.148:0/2226979094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:26.796 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:27.174 INFO:tasks.workunit.client.0.smithi148.stdout:write: Y 0~1 2024-03-07T03:16:27.237 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:16:27.439 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:16:27.718 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (2579 ms) 2024-03-07T03:16:27.718 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-03-07T03:16:27.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:27 smithi148 bash[21362]: cluster 2024-03-07T03:16:26.512056+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2024-03-07T03:16:27.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:27 smithi148 bash[21362]: cluster 2024-03-07T03:16:26.689281+0000 mgr.y (mgr.25052) 25 : cluster 0 pgmap v31: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.0 MiB/s wr, 768 op/s 2024-03-07T03:16:27.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:27 smithi148 bash[31149]: cluster 2024-03-07T03:16:26.512056+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2024-03-07T03:16:27.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:27 smithi148 bash[31149]: cluster 2024-03-07T03:16:26.689281+0000 mgr.y (mgr.25052) 25 : cluster 0 pgmap v31: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.0 MiB/s wr, 768 op/s 2024-03-07T03:16:27.789 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:27.812 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 41943040~10 2024-03-07T03:16:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:27 smithi195 bash[23221]: cluster 2024-03-07T03:16:26.512056+0000 mon.a (mon.0) 1177 : cluster 0 osdmap e244: 8 total, 8 up, 8 in 2024-03-07T03:16:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:27 smithi195 bash[23221]: cluster 2024-03-07T03:16:26.689281+0000 mgr.y (mgr.25052) 25 : cluster 0 pgmap v31: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s rd, 5.0 MiB/s wr, 768 op/s 2024-03-07T03:16:27.827 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:16:28.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:28 smithi148 bash[21362]: audit 2024-03-07T03:16:28.252425+0000 mon.a (mon.0) 1178 : audit 0 from='client.? 172.21.15.148:0/3363065612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:28.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:28 smithi148 bash[31149]: audit 2024-03-07T03:16:28.252425+0000 mon.a (mon.0) 1178 : audit 0 from='client.? 172.21.15.148:0/3363065612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:28.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:28 smithi195 bash[23221]: audit 2024-03-07T03:16:28.252425+0000 mon.a (mon.0) 1178 : audit 0 from='client.? 172.21.15.148:0/3363065612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:29.524 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[51559]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:16:29] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-07T03:16:29.555 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 41943041~10 2024-03-07T03:16:29.569 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:29.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:29 smithi148 bash[21362]: cluster 2024-03-07T03:16:28.536143+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2024-03-07T03:16:29.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:29 smithi148 bash[21362]: cluster 2024-03-07T03:16:28.690050+0000 mgr.y (mgr.25052) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 975 KiB/s wr, 702 op/s 2024-03-07T03:16:29.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:29 smithi148 bash[31149]: cluster 2024-03-07T03:16:28.536143+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2024-03-07T03:16:29.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:29 smithi148 bash[31149]: cluster 2024-03-07T03:16:28.690050+0000 mgr.y (mgr.25052) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 975 KiB/s wr, 702 op/s 2024-03-07T03:16:29.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 systemd[1]: Stopping Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:16:29.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[23221]: cluster 2024-03-07T03:16:28.536143+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e245: 8 total, 8 up, 8 in 2024-03-07T03:16:29.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[23221]: cluster 2024-03-07T03:16:28.690050+0000 mgr.y (mgr.25052) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 8.8 MiB/s rd, 975 KiB/s wr, 702 op/s 2024-03-07T03:16:30.312 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:16:30.312 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:16:30.312 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.882Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.883Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.883Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.883Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:16:30.313 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:29 smithi195 bash[57805]: ts=2024-03-07T03:16:29.883Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:16:30.690 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:30 smithi195 bash[23221]: cluster 2024-03-07T03:16:29.545763+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2024-03-07T03:16:30.690 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:30 smithi195 bash[59044]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:16:30.691 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:30 smithi195 bash[59104]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-prometheus-a 2024-03-07T03:16:30.691 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:30 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-07T03:16:30.691 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:30 smithi195 systemd[1]: Stopped Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:16:30.691 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:30 smithi195 systemd[1]: Started Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:16:30.766 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 41943042~10 2024-03-07T03:16:30.782 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:16:30.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:30 smithi148 bash[21362]: cluster 2024-03-07T03:16:29.545763+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2024-03-07T03:16:30.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:30 smithi148 bash[31149]: cluster 2024-03-07T03:16:29.545763+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e246: 8 total, 8 up, 8 in 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: cluster 2024-03-07T03:16:30.548598+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: cluster 2024-03-07T03:16:30.690805+0000 mgr.y (mgr.25052) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 1010 KiB/s wr, 727 op/s 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: audit 2024-03-07T03:16:30.706733+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: audit 2024-03-07T03:16:30.722661+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: audit 2024-03-07T03:16:30.725066+0000 mon.a (mon.0) 1184 : audit 0 from='client.? 172.21.15.148:0/1279070885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: audit 2024-03-07T03:16:30.745678+0000 mon.c (mon.2) 99 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: audit 2024-03-07T03:16:30.746388+0000 mgr.y (mgr.25052) 28 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[21362]: audit 2024-03-07T03:16:30.753706+0000 mon.c (mon.2) 100 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:31.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: cluster 2024-03-07T03:16:30.548598+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2024-03-07T03:16:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: cluster 2024-03-07T03:16:30.690805+0000 mgr.y (mgr.25052) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 1010 KiB/s wr, 727 op/s 2024-03-07T03:16:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: audit 2024-03-07T03:16:30.706733+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: audit 2024-03-07T03:16:30.722661+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: audit 2024-03-07T03:16:30.725066+0000 mon.a (mon.0) 1184 : audit 0 from='client.? 172.21.15.148:0/1279070885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: audit 2024-03-07T03:16:30.745678+0000 mon.c (mon.2) 99 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:16:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: audit 2024-03-07T03:16:30.746388+0000 mgr.y (mgr.25052) 28 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:16:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:31 smithi148 bash[31149]: audit 2024-03-07T03:16:30.753706+0000 mon.c (mon.2) 100 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: cluster 2024-03-07T03:16:30.548598+0000 mon.a (mon.0) 1181 : cluster 0 osdmap e247: 8 total, 8 up, 8 in 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: cluster 2024-03-07T03:16:30.690805+0000 mgr.y (mgr.25052) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 9.2 MiB/s rd, 1010 KiB/s wr, 727 op/s 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: audit 2024-03-07T03:16:30.706733+0000 mon.a (mon.0) 1182 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: audit 2024-03-07T03:16:30.722661+0000 mon.a (mon.0) 1183 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: audit 2024-03-07T03:16:30.725066+0000 mon.a (mon.0) 1184 : audit 0 from='client.? 172.21.15.148:0/1279070885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: audit 2024-03-07T03:16:30.745678+0000 mon.c (mon.2) 99 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: audit 2024-03-07T03:16:30.746388+0000 mgr.y (mgr.25052) 28 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-07T03:16:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:31 smithi195 bash[23221]: audit 2024-03-07T03:16:30.753706+0000 mon.c (mon.2) 100 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.447Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.447Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.447Z caller=main.go:570 level=info host_details="(Linux 5.15.0-97-generic #107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024 x86_64 smithi195 (none))" 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.447Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.447Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.449Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.449Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-07T03:16:32.694 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.452Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.452Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.453Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.453Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=5.799µs 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.453Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.455Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.466Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.475Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.476Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.476Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=34.348µs wal_replay_duration=22.273907ms wbl_replay_duration=95ns total_replay_duration=22.333911ms 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.478Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.478Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.478Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.503Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=24.744877ms db_storage=905ns remote_storage=1.376µs web_handler=386ns query_engine=749ns scrape=257.669µs scrape_sd=131.017µs notify=15.554µs notify_sd=20.771µs rules=23.975851ms tracing=6.709µs 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.503Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-07T03:16:32.695 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[59134]: ts=2024-03-07T03:16:32.503Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-07T03:16:32.740 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:16:32.789 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:16:32.889 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:16:32.952 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:32 smithi195 bash[23221]: cluster 2024-03-07T03:16:31.725359+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2024-03-07T03:16:33.038 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:32 smithi148 bash[31149]: cluster 2024-03-07T03:16:31.725359+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2024-03-07T03:16:33.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:32 smithi148 bash[21362]: cluster 2024-03-07T03:16:31.725359+0000 mon.a (mon.0) 1185 : cluster 0 osdmap e248: 8 total, 8 up, 8 in 2024-03-07T03:16:33.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[59134]: ts=2024-03-07T03:16:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:33.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[23221]: cluster 2024-03-07T03:16:32.691672+0000 mgr.y (mgr.25052) 29 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 984 B/s wr, 5 op/s 2024-03-07T03:16:33.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[23221]: cluster 2024-03-07T03:16:32.736061+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2024-03-07T03:16:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[23221]: audit 2024-03-07T03:16:33.190808+0000 mon.a (mon.0) 1187 : audit 0 from='client.? 172.21.15.148:0/1047001103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[23221]: audit 2024-03-07T03:16:33.629784+0000 mon.c (mon.2) 101 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:16:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[23221]: audit 2024-03-07T03:16:33.630969+0000 mon.a (mon.0) 1188 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:16:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[23221]: audit 2024-03-07T03:16:33.634323+0000 mon.c (mon.2) 102 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:16:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:33 smithi195 bash[23221]: audit 2024-03-07T03:16:33.636573+0000 mon.c (mon.2) 103 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:33.976 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (6258 ms) 2024-03-07T03:16:33.976 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[21362]: cluster 2024-03-07T03:16:32.691672+0000 mgr.y (mgr.25052) 29 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 984 B/s wr, 5 op/s 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[21362]: cluster 2024-03-07T03:16:32.736061+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[21362]: audit 2024-03-07T03:16:33.190808+0000 mon.a (mon.0) 1187 : audit 0 from='client.? 172.21.15.148:0/1047001103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[21362]: audit 2024-03-07T03:16:33.629784+0000 mon.c (mon.2) 101 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[21362]: audit 2024-03-07T03:16:33.630969+0000 mon.a (mon.0) 1188 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[21362]: audit 2024-03-07T03:16:33.634323+0000 mon.c (mon.2) 102 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[21362]: audit 2024-03-07T03:16:33.636573+0000 mon.c (mon.2) 103 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[31149]: cluster 2024-03-07T03:16:32.691672+0000 mgr.y (mgr.25052) 29 : cluster 0 pgmap v38: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 2.4 KiB/s rd, 984 B/s wr, 5 op/s 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[31149]: cluster 2024-03-07T03:16:32.736061+0000 mon.a (mon.0) 1186 : cluster 0 osdmap e249: 8 total, 8 up, 8 in 2024-03-07T03:16:34.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[31149]: audit 2024-03-07T03:16:33.190808+0000 mon.a (mon.0) 1187 : audit 0 from='client.? 172.21.15.148:0/1047001103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:34.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[31149]: audit 2024-03-07T03:16:33.629784+0000 mon.c (mon.2) 101 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:16:34.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[31149]: audit 2024-03-07T03:16:33.630969+0000 mon.a (mon.0) 1188 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:16:34.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[31149]: audit 2024-03-07T03:16:33.634323+0000 mon.c (mon.2) 102 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:16:34.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:33 smithi148 bash[31149]: audit 2024-03-07T03:16:33.636573+0000 mon.c (mon.2) 103 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:34.054 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 41943040~10 2024-03-07T03:16:34.097 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:16:35.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:34 smithi148 bash[31149]: cephadm 2024-03-07T03:16:33.628829+0000 mgr.y (mgr.25052) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:16:35.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:34 smithi148 bash[31149]: cephadm 2024-03-07T03:16:33.639356+0000 mgr.y (mgr.25052) 31 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:16:35.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:34 smithi148 bash[31149]: cluster 2024-03-07T03:16:33.737207+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2024-03-07T03:16:35.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:34 smithi148 bash[21362]: cephadm 2024-03-07T03:16:33.628829+0000 mgr.y (mgr.25052) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:16:35.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:34 smithi148 bash[21362]: cephadm 2024-03-07T03:16:33.639356+0000 mgr.y (mgr.25052) 31 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:16:35.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:34 smithi148 bash[21362]: cluster 2024-03-07T03:16:33.737207+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2024-03-07T03:16:35.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:34 smithi195 bash[23221]: cephadm 2024-03-07T03:16:33.628829+0000 mgr.y (mgr.25052) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-03-07T03:16:35.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:34 smithi195 bash[23221]: cephadm 2024-03-07T03:16:33.639356+0000 mgr.y (mgr.25052) 31 : cephadm 1 Deploying daemon mgr.x on smithi195 2024-03-07T03:16:35.040 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:34 smithi195 bash[23221]: cluster 2024-03-07T03:16:33.737207+0000 mon.a (mon.0) 1189 : cluster 0 osdmap e250: 8 total, 8 up, 8 in 2024-03-07T03:16:35.786 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:16:35.833 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:36.015 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:35 smithi195 bash[23221]: cluster 2024-03-07T03:16:34.692496+0000 mgr.y (mgr.25052) 32 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 14 KiB/s wr, 163 op/s 2024-03-07T03:16:36.015 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:35 smithi195 bash[23221]: cluster 2024-03-07T03:16:34.764590+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2024-03-07T03:16:36.016 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:35 smithi195 bash[23221]: audit 2024-03-07T03:16:35.630427+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.148:0/2312687365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:36.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:35 smithi148 bash[21362]: cluster 2024-03-07T03:16:34.692496+0000 mgr.y (mgr.25052) 32 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 14 KiB/s wr, 163 op/s 2024-03-07T03:16:36.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:35 smithi148 bash[21362]: cluster 2024-03-07T03:16:34.764590+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2024-03-07T03:16:36.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:35 smithi148 bash[21362]: audit 2024-03-07T03:16:35.630427+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.148:0/2312687365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:36.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:35 smithi148 bash[31149]: cluster 2024-03-07T03:16:34.692496+0000 mgr.y (mgr.25052) 32 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 14 KiB/s wr, 163 op/s 2024-03-07T03:16:36.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:35 smithi148 bash[31149]: cluster 2024-03-07T03:16:34.764590+0000 mon.a (mon.0) 1190 : cluster 0 osdmap e251: 8 total, 8 up, 8 in 2024-03-07T03:16:36.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:35 smithi148 bash[31149]: audit 2024-03-07T03:16:35.630427+0000 mon.c (mon.2) 104 : audit 0 from='client.? 172.21.15.148:0/2312687365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:36.303 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.303 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.303 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.304 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.304 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.304 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.305 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.305 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.305 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:36.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:36 smithi195 systemd[1]: Stopping Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:16:37.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:36 smithi148 bash[21362]: cluster 2024-03-07T03:16:35.779398+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2024-03-07T03:16:37.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:36 smithi148 bash[31149]: cluster 2024-03-07T03:16:35.779398+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2024-03-07T03:16:37.250 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:37.270 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:36 smithi195 bash[23221]: cluster 2024-03-07T03:16:35.779398+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e252: 8 total, 8 up, 8 in 2024-03-07T03:16:37.271 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[59417]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:16:37.277 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:16:37.550 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.551 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.551 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.551 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.551 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.552 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.552 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.552 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.552 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:16:37.552 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[59479]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:16:37.553 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-07T03:16:37.553 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: Stopped Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:16:37.553 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:37.553 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:37 smithi195 systemd[1]: Started Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:16:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[23221]: cluster 2024-03-07T03:16:36.693406+0000 mgr.y (mgr.25052) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 14 KiB/s wr, 163 op/s 2024-03-07T03:16:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[23221]: cluster 2024-03-07T03:16:36.786413+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2024-03-07T03:16:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[23221]: audit 2024-03-07T03:16:36.812972+0000 mon.c (mon.2) 105 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[23221]: audit 2024-03-07T03:16:37.616823+0000 mon.a (mon.0) 1193 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[23221]: audit 2024-03-07T03:16:37.632353+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:37 smithi195 bash[23221]: audit 2024-03-07T03:16:37.664367+0000 mon.c (mon.2) 106 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[21362]: cluster 2024-03-07T03:16:36.693406+0000 mgr.y (mgr.25052) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 14 KiB/s wr, 163 op/s 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[21362]: cluster 2024-03-07T03:16:36.786413+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[21362]: audit 2024-03-07T03:16:36.812972+0000 mon.c (mon.2) 105 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[21362]: audit 2024-03-07T03:16:37.616823+0000 mon.a (mon.0) 1193 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[21362]: audit 2024-03-07T03:16:37.632353+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[21362]: audit 2024-03-07T03:16:37.664367+0000 mon.c (mon.2) 106 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[31149]: cluster 2024-03-07T03:16:36.693406+0000 mgr.y (mgr.25052) 33 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 120 KiB/s rd, 14 KiB/s wr, 163 op/s 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[31149]: cluster 2024-03-07T03:16:36.786413+0000 mon.a (mon.0) 1192 : cluster 0 osdmap e253: 8 total, 8 up, 8 in 2024-03-07T03:16:38.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[31149]: audit 2024-03-07T03:16:36.812972+0000 mon.c (mon.2) 105 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:38.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[31149]: audit 2024-03-07T03:16:37.616823+0000 mon.a (mon.0) 1193 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:38.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[31149]: audit 2024-03-07T03:16:37.632353+0000 mon.a (mon.0) 1194 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:38.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:37 smithi148 bash[31149]: audit 2024-03-07T03:16:37.664367+0000 mon.c (mon.2) 106 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:38.802 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 41943040~10 2024-03-07T03:16:38.811 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap4 2024-03-07T03:16:39.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:38 smithi195 bash[23221]: cluster 2024-03-07T03:16:37.787525+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2024-03-07T03:16:39.026 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:38 smithi195 bash[23221]: audit 2024-03-07T03:16:38.083920+0000 mon.a (mon.0) 1196 : audit 0 from='client.? 172.21.15.148:0/1992215650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:39.026 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:38 smithi195 bash[59535]: debug 2024-03-07T03:16:38.892+0000 7f58b91c5700 1 -- 172.21.15.195:0/1562654449 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55be6f36e5a0 con 0x55be700f2800 2024-03-07T03:16:39.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:38 smithi148 bash[21362]: cluster 2024-03-07T03:16:37.787525+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2024-03-07T03:16:39.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:38 smithi148 bash[21362]: audit 2024-03-07T03:16:38.083920+0000 mon.a (mon.0) 1196 : audit 0 from='client.? 172.21.15.148:0/1992215650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:39.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:38 smithi148 bash[31149]: cluster 2024-03-07T03:16:37.787525+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e254: 8 total, 8 up, 8 in 2024-03-07T03:16:39.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:38 smithi148 bash[31149]: audit 2024-03-07T03:16:38.083920+0000 mon.a (mon.0) 1196 : audit 0 from='client.? 172.21.15.148:0/1992215650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:39.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:39 smithi195 bash[59535]: debug 2024-03-07T03:16:39.036+0000 7f58c33c7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:16:40.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:39 smithi148 bash[21362]: cluster 2024-03-07T03:16:38.694255+0000 mgr.y (mgr.25052) 34 : cluster 0 pgmap v47: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-03-07T03:16:40.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:39 smithi148 bash[21362]: cluster 2024-03-07T03:16:38.800774+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2024-03-07T03:16:40.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:39 smithi148 bash[31149]: cluster 2024-03-07T03:16:38.694255+0000 mgr.y (mgr.25052) 34 : cluster 0 pgmap v47: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-03-07T03:16:40.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:39 smithi148 bash[31149]: cluster 2024-03-07T03:16:38.800774+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2024-03-07T03:16:40.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:39 smithi195 bash[23221]: cluster 2024-03-07T03:16:38.694255+0000 mgr.y (mgr.25052) 34 : cluster 0 pgmap v47: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-03-07T03:16:40.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:39 smithi195 bash[23221]: cluster 2024-03-07T03:16:38.800774+0000 mon.a (mon.0) 1197 : cluster 0 osdmap e255: 8 total, 8 up, 8 in 2024-03-07T03:16:40.825 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 41943040 2024-03-07T03:16:40.876 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:16:41.027 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:16:41.035 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:40 smithi195 bash[23221]: cluster 2024-03-07T03:16:39.809997+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2024-03-07T03:16:41.035 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:40 smithi195 bash[23221]: audit 2024-03-07T03:16:40.521735+0000 mon.a (mon.0) 1199 : audit 0 from='client.? 172.21.15.148:0/506400383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:41.035 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:40 smithi195 bash[59535]: debug 2024-03-07T03:16:40.676+0000 7f58c33c7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:16:41.036 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:40 smithi195 bash[59535]: debug 2024-03-07T03:16:40.772+0000 7f58c33c7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:16:41.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:40 smithi148 bash[21362]: cluster 2024-03-07T03:16:39.809997+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2024-03-07T03:16:41.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:40 smithi148 bash[21362]: audit 2024-03-07T03:16:40.521735+0000 mon.a (mon.0) 1199 : audit 0 from='client.? 172.21.15.148:0/506400383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:41.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:40 smithi148 bash[31149]: cluster 2024-03-07T03:16:39.809997+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e256: 8 total, 8 up, 8 in 2024-03-07T03:16:41.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:40 smithi148 bash[31149]: audit 2024-03-07T03:16:40.521735+0000 mon.a (mon.0) 1199 : audit 0 from='client.? 172.21.15.148:0/506400383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:42.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:41 smithi195 bash[23221]: cluster 2024-03-07T03:16:40.695070+0000 mgr.y (mgr.25052) 35 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-03-07T03:16:42.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:41 smithi195 bash[23221]: cluster 2024-03-07T03:16:40.820041+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2024-03-07T03:16:42.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:41 smithi195 bash[59535]: debug 2024-03-07T03:16:41.048+0000 7f58c33c7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:16:42.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:41 smithi195 bash[59535]: debug 2024-03-07T03:16:41.872+0000 7f58c33c7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:16:42.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:41 smithi195 bash[59535]: debug 2024-03-07T03:16:41.996+0000 7f58c33c7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:16:42.086 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (8110 ms) 2024-03-07T03:16:42.086 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-03-07T03:16:42.166 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:42.189 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:42.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:41 smithi148 bash[21362]: cluster 2024-03-07T03:16:40.695070+0000 mgr.y (mgr.25052) 35 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-03-07T03:16:42.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:41 smithi148 bash[21362]: cluster 2024-03-07T03:16:40.820041+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2024-03-07T03:16:42.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:41 smithi148 bash[31149]: cluster 2024-03-07T03:16:40.695070+0000 mgr.y (mgr.25052) 35 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 277 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-03-07T03:16:42.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:41 smithi148 bash[31149]: cluster 2024-03-07T03:16:40.820041+0000 mon.a (mon.0) 1200 : cluster 0 osdmap e257: 8 total, 8 up, 8 in 2024-03-07T03:16:42.468 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:42 smithi195 bash[59535]: debug 2024-03-07T03:16:42.100+0000 7f58c33c7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:16:42.622 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.AbortInUseImage (536 ms) 2024-03-07T03:16:42.622 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-03-07T03:16:42.674 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:42.688 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:16:42.747 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:16:42.803 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:42.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:42 smithi195 bash[59535]: debug 2024-03-07T03:16:42.480+0000 7f58c33c7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:16:42.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:42 smithi195 bash[59535]: debug 2024-03-07T03:16:42.604+0000 7f58c33c7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:16:42.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:42 smithi195 bash[59535]: debug 2024-03-07T03:16:42.728+0000 7f58c33c7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:16:43.055 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:16:43.108 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:16:43.163 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:42 smithi195 bash[23221]: cluster 2024-03-07T03:16:41.836120+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2024-03-07T03:16:43.187 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:16:43.288 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:42 smithi148 bash[21362]: cluster 2024-03-07T03:16:41.836120+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2024-03-07T03:16:43.288 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:42 smithi148 bash[31149]: cluster 2024-03-07T03:16:41.836120+0000 mon.a (mon.0) 1201 : cluster 0 osdmap e258: 8 total, 8 up, 8 in 2024-03-07T03:16:43.455 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[59535]: debug 2024-03-07T03:16:43.176+0000 7f58c33c7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:16:43.497 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (875 ms) 2024-03-07T03:16:43.497 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-03-07T03:16:43.571 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:43.597 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:16:43.691 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:16:43.750 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[59535]: debug 2024-03-07T03:16:43.468+0000 7f58c33c7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:16:43.750 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[59134]: ts=2024-03-07T03:16:43.505Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:44.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[59535]: debug 2024-03-07T03:16:43.760+0000 7f58c33c7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:16:44.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[59535]: debug 2024-03-07T03:16:43.880+0000 7f58c33c7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:16:44.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[23221]: cluster 2024-03-07T03:16:42.695551+0000 mgr.y (mgr.25052) 36 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.0 KiB/s wr, 36 op/s 2024-03-07T03:16:44.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[23221]: audit 2024-03-07T03:16:43.016634+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.148:0/536118415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:44.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[23221]: cluster 2024-03-07T03:16:43.353832+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2024-03-07T03:16:44.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[23221]: audit 2024-03-07T03:16:43.763778+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:44.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:43 smithi195 bash[23221]: audit 2024-03-07T03:16:43.780419+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[31149]: cluster 2024-03-07T03:16:42.695551+0000 mgr.y (mgr.25052) 36 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.0 KiB/s wr, 36 op/s 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[31149]: audit 2024-03-07T03:16:43.016634+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.148:0/536118415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[31149]: cluster 2024-03-07T03:16:43.353832+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[31149]: audit 2024-03-07T03:16:43.763778+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[31149]: audit 2024-03-07T03:16:43.780419+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[21362]: cluster 2024-03-07T03:16:42.695551+0000 mgr.y (mgr.25052) 36 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 3.0 KiB/s wr, 36 op/s 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[21362]: audit 2024-03-07T03:16:43.016634+0000 mon.c (mon.2) 107 : audit 0 from='client.? 172.21.15.148:0/536118415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[21362]: cluster 2024-03-07T03:16:43.353832+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e259: 8 total, 8 up, 8 in 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[21362]: audit 2024-03-07T03:16:43.763778+0000 mon.a (mon.0) 1203 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:44.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:43 smithi148 bash[21362]: audit 2024-03-07T03:16:43.780419+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:44.409 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:44 smithi195 bash[59535]: debug 2024-03-07T03:16:44.168+0000 7f58c33c7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:16:44.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:44 smithi195 bash[59535]: debug 2024-03-07T03:16:44.420+0000 7f58c33c7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:16:45.230 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:44 smithi195 bash[59535]: debug 2024-03-07T03:16:44.852+0000 7f58c33c7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:16:45.230 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:44 smithi195 bash[59535]: debug 2024-03-07T03:16:44.976+0000 7f58c33c7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:16:45.383 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 8388609~8388609 2024-03-07T03:16:45.419 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 0~1000 2024-03-07T03:16:45.419 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1010~1000 2024-03-07T03:16:45.419 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:16:45.561 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:45 smithi195 bash[59535]: debug 2024-03-07T03:16:45.240+0000 7f58c33c7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:16:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:45 smithi195 bash[23221]: cluster 2024-03-07T03:16:44.366739+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2024-03-07T03:16:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:45 smithi195 bash[23221]: cluster 2024-03-07T03:16:44.696333+0000 mgr.y (mgr.25052) 37 : cluster 0 pgmap v56: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 300 KiB/s rd, 22 KiB/s wr, 396 op/s 2024-03-07T03:16:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:45 smithi195 bash[23221]: audit 2024-03-07T03:16:44.888654+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:45 smithi195 bash[23221]: audit 2024-03-07T03:16:44.904927+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[21362]: cluster 2024-03-07T03:16:44.366739+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2024-03-07T03:16:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[21362]: cluster 2024-03-07T03:16:44.696333+0000 mgr.y (mgr.25052) 37 : cluster 0 pgmap v56: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 300 KiB/s rd, 22 KiB/s wr, 396 op/s 2024-03-07T03:16:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[21362]: audit 2024-03-07T03:16:44.888654+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[21362]: audit 2024-03-07T03:16:44.904927+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:45.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[31149]: cluster 2024-03-07T03:16:44.366739+0000 mon.a (mon.0) 1205 : cluster 0 osdmap e260: 8 total, 8 up, 8 in 2024-03-07T03:16:45.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[31149]: cluster 2024-03-07T03:16:44.696333+0000 mgr.y (mgr.25052) 37 : cluster 0 pgmap v56: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 300 KiB/s rd, 22 KiB/s wr, 396 op/s 2024-03-07T03:16:45.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[31149]: audit 2024-03-07T03:16:44.888654+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:45.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:45 smithi148 bash[31149]: audit 2024-03-07T03:16:44.904927+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:46.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:46 smithi195 bash[59535]: debug 2024-03-07T03:16:46.284+0000 7f58c33c7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:16:46.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:46 smithi195 bash[59535]: debug 2024-03-07T03:16:46.408+0000 7f58c33c7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:16:46.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:46 smithi195 bash[23221]: cluster 2024-03-07T03:16:45.376130+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2024-03-07T03:16:46.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:46 smithi195 bash[23221]: audit 2024-03-07T03:16:45.494710+0000 mon.a (mon.0) 1209 : audit 0 from='client.? 172.21.15.148:0/2937325748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:46 smithi148 bash[21362]: cluster 2024-03-07T03:16:45.376130+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2024-03-07T03:16:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:46 smithi148 bash[21362]: audit 2024-03-07T03:16:45.494710+0000 mon.a (mon.0) 1209 : audit 0 from='client.? 172.21.15.148:0/2937325748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:46 smithi148 bash[31149]: cluster 2024-03-07T03:16:45.376130+0000 mon.a (mon.0) 1208 : cluster 0 osdmap e261: 8 total, 8 up, 8 in 2024-03-07T03:16:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:46 smithi148 bash[31149]: audit 2024-03-07T03:16:45.494710+0000 mon.a (mon.0) 1209 : audit 0 from='client.? 172.21.15.148:0/2937325748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:47.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:46 smithi195 bash[59535]: debug 2024-03-07T03:16:46.708+0000 7f58c33c7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:16:47.062 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:46 smithi195 bash[59535]: debug 2024-03-07T03:16:46.824+0000 7f58c33c7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:16:47.402 INFO:tasks.workunit.client.0.smithi148.stdout:write: X 8388609~1000 2024-03-07T03:16:47.416 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16777228~1000 2024-03-07T03:16:47.416 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 167772160 2024-03-07T03:16:47.437 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 83886079~8388609 2024-03-07T03:16:47.485 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:16:47.529 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[23221]: cluster 2024-03-07T03:16:46.378593+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2024-03-07T03:16:47.529 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[23221]: cluster 2024-03-07T03:16:46.697156+0000 mgr.y (mgr.25052) 38 : cluster 0 pgmap v59: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 271 KiB/s rd, 19 KiB/s wr, 359 op/s 2024-03-07T03:16:47.529 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[59535]: debug 2024-03-07T03:16:47.280+0000 7f58c33c7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:16:47.530 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[59535]: debug 2024-03-07T03:16:47.408+0000 7f58c33c7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:16:47.530 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[59535]: [07/Mar/2024:03:16:47] ENGINE Bus STARTING 2024-03-07T03:16:47.538 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:16:47] "GET /metrics HTTP/1.1" 200 38768 "" "Prometheus/2.43.0" 2024-03-07T03:16:47.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:47 smithi148 bash[21362]: cluster 2024-03-07T03:16:46.378593+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2024-03-07T03:16:47.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:47 smithi148 bash[21362]: cluster 2024-03-07T03:16:46.697156+0000 mgr.y (mgr.25052) 38 : cluster 0 pgmap v59: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 271 KiB/s rd, 19 KiB/s wr, 359 op/s 2024-03-07T03:16:47.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:47 smithi148 bash[31149]: cluster 2024-03-07T03:16:46.378593+0000 mon.a (mon.0) 1210 : cluster 0 osdmap e262: 8 total, 8 up, 8 in 2024-03-07T03:16:47.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:47 smithi148 bash[31149]: cluster 2024-03-07T03:16:46.697156+0000 mgr.y (mgr.25052) 38 : cluster 0 pgmap v59: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 271 KiB/s rd, 19 KiB/s wr, 359 op/s 2024-03-07T03:16:47.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[59535]: CherryPy Checker: 2024-03-07T03:16:47.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[59535]: The Application mounted at '' has an empty config. 2024-03-07T03:16:47.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[59535]: [07/Mar/2024:03:16:47] ENGINE Serving on http://:::9283 2024-03-07T03:16:47.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:16:47 smithi195 bash[59535]: [07/Mar/2024:03:16:47] ENGINE Bus STARTED 2024-03-07T03:16:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: cluster 2024-03-07T03:16:47.398563+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2024-03-07T03:16:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: cluster 2024-03-07T03:16:47.437022+0000 mon.a (mon.0) 1212 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:16:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: audit 2024-03-07T03:16:47.437069+0000 mon.b (mon.1) 196 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:16:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: cluster 2024-03-07T03:16:47.437401+0000 mon.a (mon.0) 1213 : cluster 0 Standby manager daemon x started 2024-03-07T03:16:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: audit 2024-03-07T03:16:47.438833+0000 mon.b (mon.1) 197 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:48.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: audit 2024-03-07T03:16:47.441060+0000 mon.b (mon.1) 198 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: audit 2024-03-07T03:16:47.442374+0000 mon.b (mon.1) 199 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[21362]: audit 2024-03-07T03:16:47.938926+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.148:0/801067265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: cluster 2024-03-07T03:16:47.398563+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: cluster 2024-03-07T03:16:47.437022+0000 mon.a (mon.0) 1212 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: audit 2024-03-07T03:16:47.437069+0000 mon.b (mon.1) 196 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: cluster 2024-03-07T03:16:47.437401+0000 mon.a (mon.0) 1213 : cluster 0 Standby manager daemon x started 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: audit 2024-03-07T03:16:47.438833+0000 mon.b (mon.1) 197 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: audit 2024-03-07T03:16:47.441060+0000 mon.b (mon.1) 198 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: audit 2024-03-07T03:16:47.442374+0000 mon.b (mon.1) 199 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:48.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:48 smithi148 bash[31149]: audit 2024-03-07T03:16:47.938926+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.148:0/801067265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: cluster 2024-03-07T03:16:47.398563+0000 mon.a (mon.0) 1211 : cluster 0 osdmap e263: 8 total, 8 up, 8 in 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: cluster 2024-03-07T03:16:47.437022+0000 mon.a (mon.0) 1212 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: audit 2024-03-07T03:16:47.437069+0000 mon.b (mon.1) 196 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: cluster 2024-03-07T03:16:47.437401+0000 mon.a (mon.0) 1213 : cluster 0 Standby manager daemon x started 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: audit 2024-03-07T03:16:47.438833+0000 mon.b (mon.1) 197 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: audit 2024-03-07T03:16:47.441060+0000 mon.b (mon.1) 198 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: audit 2024-03-07T03:16:47.442374+0000 mon.b (mon.1) 199 : audit 0 from='mgr.? 172.21.15.195:0/783453909' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:16:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:48 smithi195 bash[23221]: audit 2024-03-07T03:16:47.938926+0000 mon.c (mon.2) 108 : audit 0 from='client.? 172.21.15.148:0/801067265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:49.430 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:49.479 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 83886079~1 2024-03-07T03:16:49.500 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:16:49.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:49 smithi148 bash[21362]: cluster 2024-03-07T03:16:48.400574+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2024-03-07T03:16:49.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:49 smithi148 bash[21362]: cluster 2024-03-07T03:16:48.421563+0000 mon.a (mon.0) 1215 : cluster 0 mgrmap e31: y(active, since 41s), standbys: x 2024-03-07T03:16:49.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:49 smithi148 bash[21362]: cluster 2024-03-07T03:16:48.697962+0000 mgr.y (mgr.25052) 39 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 347 op/s 2024-03-07T03:16:49.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:49 smithi148 bash[31149]: cluster 2024-03-07T03:16:48.400574+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2024-03-07T03:16:49.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:49 smithi148 bash[31149]: cluster 2024-03-07T03:16:48.421563+0000 mon.a (mon.0) 1215 : cluster 0 mgrmap e31: y(active, since 41s), standbys: x 2024-03-07T03:16:49.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:49 smithi148 bash[31149]: cluster 2024-03-07T03:16:48.697962+0000 mgr.y (mgr.25052) 39 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 347 op/s 2024-03-07T03:16:49.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:49 smithi195 bash[23221]: cluster 2024-03-07T03:16:48.400574+0000 mon.a (mon.0) 1214 : cluster 0 osdmap e264: 8 total, 8 up, 8 in 2024-03-07T03:16:49.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:49 smithi195 bash[23221]: cluster 2024-03-07T03:16:48.421563+0000 mon.a (mon.0) 1215 : cluster 0 mgrmap e31: y(active, since 41s), standbys: x 2024-03-07T03:16:49.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:49 smithi195 bash[23221]: cluster 2024-03-07T03:16:48.697962+0000 mgr.y (mgr.25052) 39 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 347 op/s 2024-03-07T03:16:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:50 smithi148 bash[21362]: cluster 2024-03-07T03:16:49.426790+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2024-03-07T03:16:50.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:50 smithi148 bash[21362]: audit 2024-03-07T03:16:50.415899+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.148:0/2452540589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:50 smithi148 bash[31149]: cluster 2024-03-07T03:16:49.426790+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2024-03-07T03:16:50.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:50 smithi148 bash[31149]: audit 2024-03-07T03:16:50.415899+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.148:0/2452540589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:50.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:50 smithi195 bash[23221]: cluster 2024-03-07T03:16:49.426790+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e265: 8 total, 8 up, 8 in 2024-03-07T03:16:50.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:50 smithi195 bash[23221]: audit 2024-03-07T03:16:50.415899+0000 mon.c (mon.2) 109 : audit 0 from='client.? 172.21.15.148:0/2452540589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:51.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:51 smithi148 bash[21362]: cluster 2024-03-07T03:16:50.424855+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2024-03-07T03:16:51.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:51 smithi148 bash[21362]: cluster 2024-03-07T03:16:50.698788+0000 mgr.y (mgr.25052) 40 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 347 op/s 2024-03-07T03:16:51.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:51 smithi148 bash[31149]: cluster 2024-03-07T03:16:50.424855+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2024-03-07T03:16:51.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:51 smithi148 bash[31149]: cluster 2024-03-07T03:16:50.698788+0000 mgr.y (mgr.25052) 40 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 347 op/s 2024-03-07T03:16:51.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:51 smithi195 bash[23221]: cluster 2024-03-07T03:16:50.424855+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e266: 8 total, 8 up, 8 in 2024-03-07T03:16:51.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:51 smithi195 bash[23221]: cluster 2024-03-07T03:16:50.698788+0000 mgr.y (mgr.25052) 40 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 33 MiB/s wr, 347 op/s 2024-03-07T03:16:52.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:52 smithi148 bash[21362]: cluster 2024-03-07T03:16:51.440715+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2024-03-07T03:16:52.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:52 smithi148 bash[21362]: audit 2024-03-07T03:16:51.813353+0000 mon.c (mon.2) 110 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:52.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:52 smithi148 bash[31149]: cluster 2024-03-07T03:16:51.440715+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2024-03-07T03:16:52.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:52 smithi148 bash[31149]: audit 2024-03-07T03:16:51.813353+0000 mon.c (mon.2) 110 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:52.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:52 smithi195 bash[23221]: cluster 2024-03-07T03:16:51.440715+0000 mon.a (mon.0) 1218 : cluster 0 osdmap e267: 8 total, 8 up, 8 in 2024-03-07T03:16:52.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:52 smithi195 bash[23221]: audit 2024-03-07T03:16:51.813353+0000 mon.c (mon.2) 110 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:16:53.176 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:16:53.328 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:16:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[21362]: cluster 2024-03-07T03:16:52.458181+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2024-03-07T03:16:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[21362]: cluster 2024-03-07T03:16:52.700323+0000 mgr.y (mgr.25052) 41 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 KiB/s wr, 24 op/s 2024-03-07T03:16:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[21362]: audit 2024-03-07T03:16:52.901106+0000 mon.a (mon.0) 1220 : audit 0 from='client.? 172.21.15.148:0/4015999169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[21362]: cluster 2024-03-07T03:16:53.358493+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2024-03-07T03:16:53.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[31149]: cluster 2024-03-07T03:16:52.458181+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2024-03-07T03:16:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[31149]: cluster 2024-03-07T03:16:52.700323+0000 mgr.y (mgr.25052) 41 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 KiB/s wr, 24 op/s 2024-03-07T03:16:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[31149]: audit 2024-03-07T03:16:52.901106+0000 mon.a (mon.0) 1220 : audit 0 from='client.? 172.21.15.148:0/4015999169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:53 smithi148 bash[31149]: cluster 2024-03-07T03:16:53.358493+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2024-03-07T03:16:53.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:53 smithi195 bash[23221]: cluster 2024-03-07T03:16:52.458181+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e268: 8 total, 8 up, 8 in 2024-03-07T03:16:53.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:53 smithi195 bash[23221]: cluster 2024-03-07T03:16:52.700323+0000 mgr.y (mgr.25052) 41 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 2.0 MiB/s rd, 4.7 KiB/s wr, 24 op/s 2024-03-07T03:16:53.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:53 smithi195 bash[23221]: audit 2024-03-07T03:16:52.901106+0000 mon.a (mon.0) 1220 : audit 0 from='client.? 172.21.15.148:0/4015999169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:53 smithi195 bash[23221]: cluster 2024-03-07T03:16:53.358493+0000 mon.a (mon.0) 1221 : cluster 0 osdmap e269: 8 total, 8 up, 8 in 2024-03-07T03:16:53.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:16:53 smithi195 bash[59134]: ts=2024-03-07T03:16:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:16:54.385 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 0~8388609 2024-03-07T03:16:54.430 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 16777218~8388609 2024-03-07T03:16:54.545 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:16:55.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: cluster 2024-03-07T03:16:54.378842+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: cluster 2024-03-07T03:16:54.700889+0000 mgr.y (mgr.25052) 42 : cluster 0 pgmap v71: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 12 KiB/s wr, 191 op/s 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.070195+0000 mon.a (mon.0) 1223 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.078170+0000 mon.a (mon.0) 1224 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.079885+0000 mon.c (mon.2) 111 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.080746+0000 mon.c (mon.2) 112 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.086352+0000 mon.a (mon.0) 1225 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.146465+0000 mon.c (mon.2) 113 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.148770+0000 mon.c (mon.2) 114 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.157851+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.159134+0000 mon.c (mon.2) 115 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.159578+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.167157+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.168238+0000 mon.c (mon.2) 116 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.168646+0000 mon.a (mon.0) 1229 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:16:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.176021+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.177596+0000 mon.c (mon.2) 117 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.178726+0000 mon.c (mon.2) 118 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[21362]: audit 2024-03-07T03:16:55.353276+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.148:0/3625466415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: cluster 2024-03-07T03:16:54.378842+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: cluster 2024-03-07T03:16:54.700889+0000 mgr.y (mgr.25052) 42 : cluster 0 pgmap v71: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 12 KiB/s wr, 191 op/s 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.070195+0000 mon.a (mon.0) 1223 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.078170+0000 mon.a (mon.0) 1224 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.079885+0000 mon.c (mon.2) 111 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.080746+0000 mon.c (mon.2) 112 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.086352+0000 mon.a (mon.0) 1225 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.146465+0000 mon.c (mon.2) 113 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.148770+0000 mon.c (mon.2) 114 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:16:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.157851+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.159134+0000 mon.c (mon.2) 115 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.159578+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.167157+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.168238+0000 mon.c (mon.2) 116 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.168646+0000 mon.a (mon.0) 1229 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.176021+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.177596+0000 mon.c (mon.2) 117 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.178726+0000 mon.c (mon.2) 118 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-07T03:16:55.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:55 smithi148 bash[31149]: audit 2024-03-07T03:16:55.353276+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.148:0/3625466415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: cluster 2024-03-07T03:16:54.378842+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e270: 8 total, 8 up, 8 in 2024-03-07T03:16:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: cluster 2024-03-07T03:16:54.700889+0000 mgr.y (mgr.25052) 42 : cluster 0 pgmap v71: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 11 MiB/s rd, 12 KiB/s wr, 191 op/s 2024-03-07T03:16:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.070195+0000 mon.a (mon.0) 1223 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.078170+0000 mon.a (mon.0) 1224 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.079885+0000 mon.c (mon.2) 111 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.080746+0000 mon.c (mon.2) 112 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:16:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.086352+0000 mon.a (mon.0) 1225 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.146465+0000 mon.c (mon.2) 113 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.148770+0000 mon.c (mon.2) 114 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.157851+0000 mon.a (mon.0) 1226 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.159134+0000 mon.c (mon.2) 115 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.159578+0000 mon.a (mon.0) 1227 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.167157+0000 mon.a (mon.0) 1228 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.168238+0000 mon.c (mon.2) 116 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.168646+0000 mon.a (mon.0) 1229 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.176021+0000 mon.a (mon.0) 1230 : audit 1 from='mgr.25052 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.177596+0000 mon.c (mon.2) 117 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.178726+0000 mon.c (mon.2) 118 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-07T03:16:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:55 smithi195 bash[23221]: audit 2024-03-07T03:16:55.353276+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.148:0/3625466415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:56.763 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:56 smithi148 bash[21362]: cephadm 2024-03-07T03:16:55.149611+0000 mgr.y (mgr.25052) 43 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-03-07T03:16:56.763 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:56 smithi148 bash[21362]: cephadm 2024-03-07T03:16:55.179375+0000 mgr.y (mgr.25052) 44 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-03-07T03:16:56.763 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:56 smithi148 bash[21362]: cluster 2024-03-07T03:16:55.385691+0000 mon.a (mon.0) 1231 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2024-03-07T03:16:56.764 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:56 smithi148 bash[31149]: cephadm 2024-03-07T03:16:55.149611+0000 mgr.y (mgr.25052) 43 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-03-07T03:16:56.764 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:56 smithi148 bash[31149]: cephadm 2024-03-07T03:16:55.179375+0000 mgr.y (mgr.25052) 44 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-03-07T03:16:56.764 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:56 smithi148 bash[31149]: cluster 2024-03-07T03:16:55.385691+0000 mon.a (mon.0) 1231 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2024-03-07T03:16:56.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:56 smithi195 bash[23221]: cephadm 2024-03-07T03:16:55.149611+0000 mgr.y (mgr.25052) 43 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-03-07T03:16:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:56 smithi195 bash[23221]: cephadm 2024-03-07T03:16:55.179375+0000 mgr.y (mgr.25052) 44 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-03-07T03:16:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:56 smithi195 bash[23221]: cluster 2024-03-07T03:16:55.385691+0000 mon.a (mon.0) 1231 : cluster 0 osdmap e271: 8 total, 8 up, 8 in 2024-03-07T03:16:57.538 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:16:57] "GET /metrics HTTP/1.1" 200 38765 "" "Prometheus/2.43.0" 2024-03-07T03:16:57.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:57 smithi148 bash[21362]: cluster 2024-03-07T03:16:56.400724+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2024-03-07T03:16:57.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:57 smithi148 bash[21362]: cluster 2024-03-07T03:16:56.701429+0000 mgr.y (mgr.25052) 45 : cluster 0 pgmap v74: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 7.0 KiB/s wr, 167 op/s 2024-03-07T03:16:57.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:57 smithi148 bash[31149]: cluster 2024-03-07T03:16:56.400724+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2024-03-07T03:16:57.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:57 smithi148 bash[31149]: cluster 2024-03-07T03:16:56.701429+0000 mgr.y (mgr.25052) 45 : cluster 0 pgmap v74: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 7.0 KiB/s wr, 167 op/s 2024-03-07T03:16:57.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:57 smithi195 bash[23221]: cluster 2024-03-07T03:16:56.400724+0000 mon.a (mon.0) 1232 : cluster 0 osdmap e272: 8 total, 8 up, 8 in 2024-03-07T03:16:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:57 smithi195 bash[23221]: cluster 2024-03-07T03:16:56.701429+0000 mgr.y (mgr.25052) 45 : cluster 0 pgmap v74: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 9.1 MiB/s rd, 7.0 KiB/s wr, 167 op/s 2024-03-07T03:16:58.769 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: cluster 2024-03-07T03:16:57.413300+0000 mon.a (mon.0) 1233 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2024-03-07T03:16:58.769 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: cephadm 2024-03-07T03:16:57.501552+0000 mgr.y (mgr.25052) 46 : cephadm 1 Upgrade: Updating mon.a 2024-03-07T03:16:58.769 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: audit 2024-03-07T03:16:57.508130+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: audit 2024-03-07T03:16:57.509428+0000 mon.c (mon.2) 120 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: audit 2024-03-07T03:16:57.510279+0000 mon.c (mon.2) 121 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: audit 2024-03-07T03:16:57.511154+0000 mon.c (mon.2) 122 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: cephadm 2024-03-07T03:16:57.512229+0000 mgr.y (mgr.25052) 47 : cephadm 1 Deploying daemon mon.a on smithi148 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[21362]: audit 2024-03-07T03:16:57.823354+0000 mon.a (mon.0) 1235 : audit 0 from='client.? 172.21.15.148:0/3000005517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: cluster 2024-03-07T03:16:57.413300+0000 mon.a (mon.0) 1233 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: cephadm 2024-03-07T03:16:57.501552+0000 mgr.y (mgr.25052) 46 : cephadm 1 Upgrade: Updating mon.a 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: audit 2024-03-07T03:16:57.508130+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: audit 2024-03-07T03:16:57.509428+0000 mon.c (mon.2) 120 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: audit 2024-03-07T03:16:57.510279+0000 mon.c (mon.2) 121 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: audit 2024-03-07T03:16:57.511154+0000 mon.c (mon.2) 122 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: cephadm 2024-03-07T03:16:57.512229+0000 mgr.y (mgr.25052) 47 : cephadm 1 Deploying daemon mon.a on smithi148 2024-03-07T03:16:58.770 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:58 smithi148 bash[31149]: audit 2024-03-07T03:16:57.823354+0000 mon.a (mon.0) 1235 : audit 0 from='client.? 172.21.15.148:0/3000005517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: cluster 2024-03-07T03:16:57.413300+0000 mon.a (mon.0) 1233 : cluster 0 osdmap e273: 8 total, 8 up, 8 in 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: cephadm 2024-03-07T03:16:57.501552+0000 mgr.y (mgr.25052) 46 : cephadm 1 Upgrade: Updating mon.a 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: audit 2024-03-07T03:16:57.508130+0000 mon.a (mon.0) 1234 : audit 1 from='mgr.25052 ' entity='mgr.y' 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: audit 2024-03-07T03:16:57.509428+0000 mon.c (mon.2) 120 : audit 1 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: audit 2024-03-07T03:16:57.510279+0000 mon.c (mon.2) 121 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: audit 2024-03-07T03:16:57.511154+0000 mon.c (mon.2) 122 : audit 0 from='mgr.25052 172.21.15.148:0/1835838303' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: cephadm 2024-03-07T03:16:57.512229+0000 mgr.y (mgr.25052) 47 : cephadm 1 Deploying daemon mon.a on smithi148 2024-03-07T03:16:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:58 smithi195 bash[23221]: audit 2024-03-07T03:16:57.823354+0000 mon.a (mon.0) 1235 : audit 0 from='client.? 172.21.15.148:0/3000005517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:16:58.934 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.AbortWithSnapshots (15437 ms) 2024-03-07T03:16:58.934 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-03-07T03:16:58.988 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:16:59.009 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:16:59.029 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:16:59.668 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:59 smithi148 bash[31149]: cluster 2024-03-07T03:16:58.415740+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2024-03-07T03:16:59.668 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:59 smithi148 bash[31149]: cluster 2024-03-07T03:16:58.702037+0000 mgr.y (mgr.25052) 48 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 23 MiB/s wr, 314 op/s 2024-03-07T03:16:59.668 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:59 smithi148 bash[21362]: cluster 2024-03-07T03:16:58.415740+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2024-03-07T03:16:59.668 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:59 smithi148 bash[21362]: cluster 2024-03-07T03:16:58.702037+0000 mgr.y (mgr.25052) 48 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 23 MiB/s wr, 314 op/s 2024-03-07T03:16:59.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:59 smithi195 bash[23221]: cluster 2024-03-07T03:16:58.415740+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e274: 8 total, 8 up, 8 in 2024-03-07T03:16:59.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:16:59 smithi195 bash[23221]: cluster 2024-03-07T03:16:58.702037+0000 mgr.y (mgr.25052) 48 : cluster 0 pgmap v77: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 31 MiB/s rd, 23 MiB/s wr, 314 op/s 2024-03-07T03:16:59.951 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.951 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.951 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.952 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.952 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.952 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.952 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.953 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:16:59.953 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:00.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:16:59 smithi148 systemd[1]: Stopping Ceph mon.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:17:00.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:00 smithi148 bash[21362]: debug 2024-03-07T03:17:00.062+0000 7f751b086700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:17:00.255 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:00 smithi148 bash[21362]: debug 2024-03-07T03:17:00.062+0000 7f751b086700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-03-07T03:17:00.884 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:00 smithi148 bash[79698]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T03:17:00.884 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:00 smithi148 bash[79762]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-a 2024-03-07T03:17:00.885 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-07T03:17:00.885 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: Stopped Ceph mon.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:17:01.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.290 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.290 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.290 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.291 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.291 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:01.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 systemd[1]: Started Ceph mon.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:17:01.291 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:17:00 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:17:02.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.957+0000 7f19620d1c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T03:17:02.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.957+0000 7f19620d1c80 0 ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev), process ceph-mon, pid 7 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.957+0000 7f19620d1c80 0 pidfile_write: ignore empty --pid-file 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 0 load: jerasure load: lrc 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Git sha 0 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Compile date 2024-03-06 19:59:24 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: DB SUMMARY 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: DB Session ID: O2KW2H34RRKA3LQB1NAQ 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1990 Bytes 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000038.sst 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000036.log size: 871852 ; 2024-03-07T03:17:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.env: 0x556f32120b20 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.info_log: 0x556f3422f920 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.statistics: (nil) 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.use_fsync: 0 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T03:17:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.db_log_dir: 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.wal_dir: 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T03:17:02.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.write_buffer_manager: 0x556f34202dc0 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T03:17:02.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T03:17:02.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T03:17:02.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.unordered_write: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.row_cache: None 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.wal_filter: None 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.wal_compression: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T03:17:02.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T03:17:02.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_open_files: -1 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Compression algorithms supported: 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kZSTD supported: 0 2024-03-07T03:17:02.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.merge_operator: 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_filter: None 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T03:17:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556f3422fa20) 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: cache_index_and_filter_blocks: 1 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: pin_top_level_index_and_filter: 1 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: index_type: 0 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: data_block_index_type: 0 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: index_shortening: 1 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: checksum: 4 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: no_block_cache: 0 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_cache: 0x556f34233090 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_cache_name: BinnedLRUCache 2024-03-07T03:17:02.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_cache_options: 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: capacity : 536870912 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: num_shard_bits : 4 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: strict_capacity_limit : 0 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: high_pri_pool_ratio: 0.000 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_cache_compressed: (nil) 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: persistent_cache: (nil) 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_size: 4096 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_size_deviation: 10 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_restart_interval: 16 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: index_block_restart_interval: 1 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: metadata_block_size: 4096 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: partition_filters: 0 2024-03-07T03:17:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: use_delta_encoding: 1 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: filter_policy: bloomfilter 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: whole_key_filtering: 1 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: verify_compression: 0 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: read_amp_bytes_per_bit: 0 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: format_version: 5 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: enable_index_compression: 1 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: block_align: 0 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: max_auto_readahead_size: 262144 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: prepopulate_block_cache: 0 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: initial_auto_readahead_size: 8192 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: num_file_reads_for_auto_readahead: 2 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression: NoCompression 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T03:17:02.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.num_levels: 7 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T03:17:02.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T03:17:02.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T03:17:02.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T03:17:02.305 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.ttl: 2592000 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T03:17:02.306 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.961+0000 7f19620d1c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.965+0000 7f19620d1c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.965+0000 7f19620d1c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 40, last_sequence is 19748, log_number is 36,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 36 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.965+0000 7f19620d1c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 36 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.965+0000 7f19620d1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 8213d278-f7a6-4983-a9aa-c0583be7d69e 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.965+0000 7f19620d1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781421968423, "job": 1, "event": "recovery_started", "wal_files": [36]} 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.965+0000 7f19620d1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #36 mode 2 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781421972896, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 41, "file_size": 836694, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19750, "largest_seqno": 20264, "table_properties": {"data_size": 833325, "index_size": 1715, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 645, "raw_key_size": 5820, "raw_average_key_size": 24, "raw_value_size": 827890, "raw_average_value_size": 3522, "num_data_blocks": 75, "num_entries": 235, "num_filter_entries": 235, "num_deletions": 5, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709781421, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "8213d278-f7a6-4983-a9aa-c0583be7d69e", "db_session_id": "O2KW2H34RRKA3LQB1NAQ", "orig_file_number": 41, "seqno_to_time_mapping": "N/A"}} 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781421973020, "job": 1, "event": "recovery_finished"} 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 43 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000036.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T03:17:02.307 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x556f3432a000 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 4 rocksdb: DB pointer 0x556f34314000 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f195083a700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f195083a700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: ** DB Stats ** 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: ** Compaction Stats [default] ** 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:17:02.308 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: L0 1/0 817.08 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 287.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: L6 1/0 10.50 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Sum 2/0 11.30 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 287.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 287.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: ** Compaction Stats [default] ** 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 287.0 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Flush(GB): cumulative 0.001, interval 0.001 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T03:17:02.309 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Cumulative compaction: 0.00 GB write, 107.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Interval compaction: 0.00 GB write, 107.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Block cache BinnedLRUCache@0x556f34233090#7 capacity: 512.00 MB usage: 2.55 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 2.1e-05 secs_since: 0 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Block cache entry stats(count,size,portion): FilterBlock(1,0.72 KB,0.000137091%) IndexBlock(1,1.83 KB,0.000348687%) Misc(1,0.00 KB,0%) 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.969+0000 7f19620d1c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] at bind addrs [v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 1 mon.a@-1(???) e3 preinit fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 0 mon.a@-1(???).mds e10 new map 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 0 mon.a@-1(???).mds e10 print_map 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: e10 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: legacy client fscid: 1 2024-03-07T03:17:02.310 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Filesystem 'foo' (1) 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: fs_name foo 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: epoch 10 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: created 2024-03-07T03:11:17.721445+0000 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: modified 2024-03-07T03:11:28.139230+0000 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: tableserver 0 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: root 0 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: session_timeout 60 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: session_autoclose 300 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: max_file_size 1099511627776 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: max_xattr_size 65536 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: required_client_features {} 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: last_failure 0 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: last_failure_osd_epoch 70 2024-03-07T03:17:02.311 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: max_mds 1 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: in 0 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: up {0=24385} 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: failed 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: damaged 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: stopped 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: data_pools [4] 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: metadata_pool 3 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: inline_data disabled 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: balancer 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: bal_rank_mask -1 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: standby_count_wanted 1 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: [mds.foo.smithi195.gtlmjn{0:24385} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: 2024-03-07T03:17:02.312 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: Standby daemons: 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: [mds.foo.smithi148.qizikc{-1:14580} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.148:6834/4101454543,v1:172.21.15.148:6835/4101454543] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 0 mon.a@-1(???).osd e275 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 0 mon.a@-1(???).osd e275 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 0 mon.a@-1(???).osd e275 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 0 mon.a@-1(???).osd e275 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:17:02.313 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:01 smithi148 bash[79819]: debug 2024-03-07T03:17:01.973+0000 7f19620d1c80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-03-07T03:17:02.561 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:02 smithi195 bash[59134]: ts=2024-03-07T03:17:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:03.504 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:03.567 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:03.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[72584]: ignoring --setuser ceph since I am not root 2024-03-07T03:17:03.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[72584]: ignoring --setgroup ceph since I am not root 2024-03-07T03:17:03.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[72584]: debug 2024-03-07T03:17:03.537+0000 7f6cfa2ba700 1 -- 172.21.15.148:0/2999960542 <== mon.0 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560018f565a0 con 0x560019cd6400 2024-03-07T03:17:03.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[72584]: debug 2024-03-07T03:17:03.689+0000 7f6d0409e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: audit 2024-03-07T03:17:00.277291+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.148:0/1861657864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:00.702692+0000 mgr.y (mgr.25052) 49 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 21 MiB/s wr, 292 op/s 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:02.190316+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:02.703642+0000 mgr.y (mgr.25052) 50 : cluster 0 pgmap v80: 137 pgs: 137 active+clean; 933 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 15 MiB/s wr, 226 op/s 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: audit 2024-03-07T03:17:02.723868+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.148:0/3521467509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.418158+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.428431+0000 mon.a (mon.0) 3 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.429661+0000 mon.a (mon.0) 4 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.429780+0000 mon.a (mon.0) 5 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.432193+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e31: y(active, since 56s), standbys: x 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.433344+0000 mon.a (mon.0) 7 : cluster 3 Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-03-07T03:17:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.433403+0000 mon.a (mon.0) 8 : cluster 3 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.433442+0000 mon.a (mon.0) 9 : cluster 3 application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.433481+0000 mon.a (mon.0) 10 : cluster 3 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: audit 2024-03-07T03:17:03.461344+0000 mon.a (mon.0) 11 : audit 1 from='mgr.25052 ' entity='' 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.462977+0000 mon.a (mon.0) 12 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: debug 2024-03-07T03:17:03.481+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: audit 2024-03-07T03:17:00.277291+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.148:0/1861657864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:00.702692+0000 mgr.y (mgr.25052) 49 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 21 MiB/s wr, 292 op/s 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:02.190316+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:02.703642+0000 mgr.y (mgr.25052) 50 : cluster 0 pgmap v80: 137 pgs: 137 active+clean; 933 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 15 MiB/s wr, 226 op/s 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: audit 2024-03-07T03:17:02.723868+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.148:0/3521467509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.418158+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:17:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.428431+0000 mon.a (mon.0) 3 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.429661+0000 mon.a (mon.0) 4 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.429780+0000 mon.a (mon.0) 5 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.432193+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e31: y(active, since 56s), standbys: x 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.433344+0000 mon.a (mon.0) 7 : cluster 3 Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.433403+0000 mon.a (mon.0) 8 : cluster 3 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.433442+0000 mon.a (mon.0) 9 : cluster 3 application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.433481+0000 mon.a (mon.0) 10 : cluster 3 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: audit 2024-03-07T03:17:03.461344+0000 mon.a (mon.0) 11 : audit 1 from='mgr.25052 ' entity='' 2024-03-07T03:17:03.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:03 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.462977+0000 mon.a (mon.0) 12 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-03-07T03:17:03.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[59535]: ignoring --setuser ceph since I am not root 2024-03-07T03:17:03.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[59535]: ignoring --setgroup ceph since I am not root 2024-03-07T03:17:03.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[59535]: debug 2024-03-07T03:17:03.540+0000 7f9b4fc7a700 1 -- 172.21.15.195:0/3357809813 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5590fbb645a0 con 0x5590fc8e4c00 2024-03-07T03:17:03.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[59535]: debug 2024-03-07T03:17:03.696+0000 7f9b59a5e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-07T03:17:03.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[59134]: ts=2024-03-07T03:17:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: audit 2024-03-07T03:17:00.277291+0000 mon.c (mon.2) 123 : audit 0 from='client.? 172.21.15.148:0/1861657864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:00.702692+0000 mgr.y (mgr.25052) 49 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 937 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 21 MiB/s wr, 292 op/s 2024-03-07T03:17:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:02.190316+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-03-07T03:17:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:02.703642+0000 mgr.y (mgr.25052) 50 : cluster 0 pgmap v80: 137 pgs: 137 active+clean; 933 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 15 MiB/s wr, 226 op/s 2024-03-07T03:17:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: audit 2024-03-07T03:17:02.723868+0000 mon.c (mon.2) 124 : audit 0 from='client.? 172.21.15.148:0/3521467509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.418158+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.428431+0000 mon.a (mon.0) 3 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.429661+0000 mon.a (mon.0) 4 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.429780+0000 mon.a (mon.0) 5 : cluster 0 osdmap e275: 8 total, 8 up, 8 in 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.432193+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e31: y(active, since 56s), standbys: x 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.433344+0000 mon.a (mon.0) 7 : cluster 3 Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.433403+0000 mon.a (mon.0) 8 : cluster 3 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.433442+0000 mon.a (mon.0) 9 : cluster 3 application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.433481+0000 mon.a (mon.0) 10 : cluster 3 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: audit 2024-03-07T03:17:03.461344+0000 mon.a (mon.0) 11 : audit 1 from='mgr.25052 ' entity='' 2024-03-07T03:17:03.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:03 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.462977+0000 mon.a (mon.0) 12 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-03-07T03:17:04.788 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:17:04.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:04 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.484503+0000 mon.a (mon.0) 13 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:17:04.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:04 smithi148 bash[31149]: cluster 2024-03-07T03:17:03.498218+0000 mon.a (mon.0) 14 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2024-03-07T03:17:04.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:04 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.484503+0000 mon.a (mon.0) 13 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:17:04.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:04 smithi148 bash[79819]: cluster 2024-03-07T03:17:03.498218+0000 mon.a (mon.0) 14 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2024-03-07T03:17:04.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:04 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.484503+0000 mon.a (mon.0) 13 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:17:04.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:04 smithi195 bash[23221]: cluster 2024-03-07T03:17:03.498218+0000 mon.a (mon.0) 14 : cluster 0 osdmap e276: 8 total, 8 up, 8 in 2024-03-07T03:17:04.927 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:17:05.170 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 1 2024-03-07T03:17:05.428 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:05 smithi148 bash[72584]: debug 2024-03-07T03:17:05.337+0000 7f6d0409e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:17:05.561 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:05 smithi195 bash[59535]: debug 2024-03-07T03:17:05.320+0000 7f9b59a5e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-07T03:17:05.562 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:05 smithi195 bash[59535]: debug 2024-03-07T03:17:05.420+0000 7f9b59a5e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:17:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:05 smithi195 bash[23221]: cluster 2024-03-07T03:17:04.505237+0000 mon.a (mon.0) 15 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2024-03-07T03:17:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:05 smithi195 bash[23221]: audit 2024-03-07T03:17:05.171816+0000 mon.a (mon.0) 16 : audit 0 from='client.? 172.21.15.148:0/3600715368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:05.702 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV1Parent (6768 ms) 2024-03-07T03:17:05.703 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-03-07T03:17:05.725 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:05 smithi148 bash[72584]: debug 2024-03-07T03:17:05.437+0000 7f6d0409e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-07T03:17:05.725 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:05 smithi148 bash[79819]: cluster 2024-03-07T03:17:04.505237+0000 mon.a (mon.0) 15 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2024-03-07T03:17:05.725 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:05 smithi148 bash[79819]: audit 2024-03-07T03:17:05.171816+0000 mon.a (mon.0) 16 : audit 0 from='client.? 172.21.15.148:0/3600715368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:05.725 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:05 smithi148 bash[31149]: cluster 2024-03-07T03:17:04.505237+0000 mon.a (mon.0) 15 : cluster 0 osdmap e277: 8 total, 8 up, 8 in 2024-03-07T03:17:05.725 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:05 smithi148 bash[31149]: audit 2024-03-07T03:17:05.171816+0000 mon.a (mon.0) 16 : audit 0 from='client.? 172.21.15.148:0/3600715368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:17:05.764 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:05.789 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:05.807 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:06.020 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:05 smithi148 bash[72584]: debug 2024-03-07T03:17:05.733+0000 7f6d0409e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:17:06.021 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:17:06.061 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:05 smithi195 bash[59535]: debug 2024-03-07T03:17:05.696+0000 7f9b59a5e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-07T03:17:06.786 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:06 smithi148 bash[79819]: cluster 2024-03-07T03:17:05.518884+0000 mon.a (mon.0) 17 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2024-03-07T03:17:06.786 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:06 smithi148 bash[31149]: cluster 2024-03-07T03:17:05.518884+0000 mon.a (mon.0) 17 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2024-03-07T03:17:06.786 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:06 smithi148 bash[72584]: debug 2024-03-07T03:17:06.577+0000 7f6d0409e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:17:06.786 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:06 smithi148 bash[72584]: debug 2024-03-07T03:17:06.689+0000 7f6d0409e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:17:06.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:06 smithi195 bash[23221]: cluster 2024-03-07T03:17:05.518884+0000 mon.a (mon.0) 17 : cluster 0 osdmap e278: 8 total, 8 up, 8 in 2024-03-07T03:17:06.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:06 smithi195 bash[59535]: debug 2024-03-07T03:17:06.504+0000 7f9b59a5e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-07T03:17:06.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:06 smithi195 bash[59535]: debug 2024-03-07T03:17:06.624+0000 7f9b59a5e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-07T03:17:06.812 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:06 smithi195 bash[59535]: debug 2024-03-07T03:17:06.732+0000 7f9b59a5e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:17:06.993 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:07.096 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:07.508 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:06 smithi148 bash[72584]: debug 2024-03-07T03:17:06.797+0000 7f6d0409e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-07T03:17:07.509 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[72584]: debug 2024-03-07T03:17:07.189+0000 7f6d0409e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:17:07.509 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[72584]: debug 2024-03-07T03:17:07.313+0000 7f6d0409e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:17:07.509 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[72584]: debug 2024-03-07T03:17:07.429+0000 7f6d0409e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:17:07.509 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:07 smithi195 bash[59535]: debug 2024-03-07T03:17:07.120+0000 7f9b59a5e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-07T03:17:07.509 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:07 smithi195 bash[59535]: debug 2024-03-07T03:17:07.244+0000 7f9b59a5e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-07T03:17:07.509 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:07 smithi195 bash[59535]: debug 2024-03-07T03:17:07.348+0000 7f9b59a5e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-07T03:17:07.784 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[31149]: cluster 2024-03-07T03:17:06.518430+0000 mon.a (mon.0) 18 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2024-03-07T03:17:07.784 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[31149]: cluster 2024-03-07T03:17:06.986392+0000 mon.a (mon.0) 19 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2024-03-07T03:17:07.785 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[79819]: cluster 2024-03-07T03:17:06.518430+0000 mon.a (mon.0) 18 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2024-03-07T03:17:07.785 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[79819]: cluster 2024-03-07T03:17:06.986392+0000 mon.a (mon.0) 19 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2024-03-07T03:17:07.811 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:07 smithi195 bash[59535]: debug 2024-03-07T03:17:07.708+0000 7f9b59a5e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:17:07.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:07 smithi195 bash[23221]: cluster 2024-03-07T03:17:06.518430+0000 mon.a (mon.0) 18 : cluster 0 osdmap e279: 8 total, 8 up, 8 in 2024-03-07T03:17:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:07 smithi195 bash[23221]: cluster 2024-03-07T03:17:06.986392+0000 mon.a (mon.0) 19 : cluster 0 osdmap e280: 8 total, 8 up, 8 in 2024-03-07T03:17:08.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:07 smithi148 bash[72584]: debug 2024-03-07T03:17:07.793+0000 7f6d0409e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-07T03:17:08.228 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:07 smithi195 bash[59535]: debug 2024-03-07T03:17:07.956+0000 7f9b59a5e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:17:08.338 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:17:08.348 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:08 smithi148 bash[72584]: debug 2024-03-07T03:17:08.049+0000 7f6d0409e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-07T03:17:08.490 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:17:08.561 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:08 smithi195 bash[59535]: debug 2024-03-07T03:17:08.240+0000 7f9b59a5e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:17:08.561 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:08 smithi195 bash[59535]: debug 2024-03-07T03:17:08.364+0000 7f9b59a5e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:17:08.773 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:08 smithi148 bash[72584]: debug 2024-03-07T03:17:08.357+0000 7f6d0409e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-07T03:17:08.773 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:08 smithi148 bash[72584]: debug 2024-03-07T03:17:08.489+0000 7f6d0409e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-07T03:17:08.984 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:08 smithi195 bash[59535]: debug 2024-03-07T03:17:08.668+0000 7f9b59a5e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:17:08.984 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:08 smithi195 bash[59535]: debug 2024-03-07T03:17:08.916+0000 7f9b59a5e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:17:09.036 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:08 smithi148 bash[72584]: debug 2024-03-07T03:17:08.781+0000 7f6d0409e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-07T03:17:09.036 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:08 smithi148 bash[31149]: cluster 2024-03-07T03:17:07.991859+0000 mon.a (mon.0) 20 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2024-03-07T03:17:09.036 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:08 smithi148 bash[79819]: cluster 2024-03-07T03:17:07.991859+0000 mon.a (mon.0) 20 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2024-03-07T03:17:09.194 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV2Parent (3492 ms) 2024-03-07T03:17:09.194 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-03-07T03:17:09.274 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:09.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:09 smithi148 bash[72584]: debug 2024-03-07T03:17:09.045+0000 7f6d0409e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:17:09.298 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:09.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:08 smithi195 bash[23221]: cluster 2024-03-07T03:17:07.991859+0000 mon.a (mon.0) 20 : cluster 0 osdmap e281: 8 total, 8 up, 8 in 2024-03-07T03:17:09.318 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:09.741 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:09 smithi195 bash[59535]: debug 2024-03-07T03:17:09.352+0000 7f9b59a5e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:17:09.741 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:09 smithi195 bash[59535]: debug 2024-03-07T03:17:09.484+0000 7f9b59a5e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:17:09.741 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:09 smithi195 bash[59535]: debug 2024-03-07T03:17:09.752+0000 7f9b59a5e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:17:09.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:09 smithi148 bash[72584]: debug 2024-03-07T03:17:09.493+0000 7f6d0409e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-07T03:17:09.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:09 smithi148 bash[72584]: debug 2024-03-07T03:17:09.613+0000 7f6d0409e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-07T03:17:10.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:10 smithi195 bash[23221]: cluster 2024-03-07T03:17:08.995186+0000 mon.a (mon.0) 21 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2024-03-07T03:17:10.289 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:09 smithi148 bash[72584]: debug 2024-03-07T03:17:09.873+0000 7f6d0409e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-07T03:17:10.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:10 smithi148 bash[31149]: cluster 2024-03-07T03:17:08.995186+0000 mon.a (mon.0) 21 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2024-03-07T03:17:10.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:09 smithi148 bash[79819]: cluster 2024-03-07T03:17:08.995186+0000 mon.a (mon.0) 21 : cluster 0 osdmap e282: 8 total, 8 up, 8 in 2024-03-07T03:17:10.742 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:17:11.019 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:11.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:11 smithi148 bash[79819]: cluster 2024-03-07T03:17:10.008873+0000 mon.a (mon.0) 22 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2024-03-07T03:17:11.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:10 smithi148 bash[72584]: debug 2024-03-07T03:17:10.933+0000 7f6d0409e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:17:11.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:11 smithi148 bash[31149]: cluster 2024-03-07T03:17:10.008873+0000 mon.a (mon.0) 22 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2024-03-07T03:17:11.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:11 smithi195 bash[23221]: cluster 2024-03-07T03:17:10.008873+0000 mon.a (mon.0) 22 : cluster 0 osdmap e283: 8 total, 8 up, 8 in 2024-03-07T03:17:11.061 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:10 smithi195 bash[59535]: debug 2024-03-07T03:17:10.780+0000 7f9b59a5e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-07T03:17:11.061 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:10 smithi195 bash[59535]: debug 2024-03-07T03:17:10.896+0000 7f9b59a5e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:17:11.131 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:11.373 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:11 smithi148 bash[72584]: debug 2024-03-07T03:17:11.061+0000 7f6d0409e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-07T03:17:11.561 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:11 smithi195 bash[59535]: debug 2024-03-07T03:17:11.224+0000 7f9b59a5e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:17:11.561 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:11 smithi195 bash[59535]: debug 2024-03-07T03:17:11.344+0000 7f9b59a5e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:17:11.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:11 smithi148 bash[72584]: debug 2024-03-07T03:17:11.381+0000 7f6d0409e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-07T03:17:11.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:11 smithi148 bash[72584]: debug 2024-03-07T03:17:11.501+0000 7f6d0409e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-07T03:17:12.056 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:11 smithi195 bash[59535]: debug 2024-03-07T03:17:11.812+0000 7f9b59a5e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:17:12.056 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:11 smithi195 bash[59535]: debug 2024-03-07T03:17:11.936+0000 7f9b59a5e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:17:12.056 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:11 smithi195 bash[59535]: [07/Mar/2024:03:17:11] ENGINE Bus STARTING 2024-03-07T03:17:12.056 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[23221]: cluster 2024-03-07T03:17:11.015853+0000 mon.a (mon.0) 23 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2024-03-07T03:17:12.057 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[23221]: cluster 2024-03-07T03:17:11.959253+0000 mon.a (mon.0) 24 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:17:12.057 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[23221]: cluster 2024-03-07T03:17:11.959379+0000 mon.a (mon.0) 25 : cluster 0 Standby manager daemon x started 2024-03-07T03:17:12.057 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[23221]: audit 2024-03-07T03:17:11.963586+0000 mon.b (mon.1) 200 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:17:12.057 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[23221]: audit 2024-03-07T03:17:11.964785+0000 mon.b (mon.1) 201 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:17:12.057 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[23221]: audit 2024-03-07T03:17:11.967700+0000 mon.b (mon.1) 202 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:17:12.057 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[23221]: audit 2024-03-07T03:17:11.969454+0000 mon.b (mon.1) 203 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:17:12.127 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:11.997+0000 7f6d0409e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[79819]: cluster 2024-03-07T03:17:11.015853+0000 mon.a (mon.0) 23 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[79819]: cluster 2024-03-07T03:17:11.959253+0000 mon.a (mon.0) 24 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[79819]: cluster 2024-03-07T03:17:11.959379+0000 mon.a (mon.0) 25 : cluster 0 Standby manager daemon x started 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[79819]: audit 2024-03-07T03:17:11.963586+0000 mon.b (mon.1) 200 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[79819]: audit 2024-03-07T03:17:11.964785+0000 mon.b (mon.1) 201 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[79819]: audit 2024-03-07T03:17:11.967700+0000 mon.b (mon.1) 202 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[79819]: audit 2024-03-07T03:17:11.969454+0000 mon.b (mon.1) 203 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[31149]: cluster 2024-03-07T03:17:11.015853+0000 mon.a (mon.0) 23 : cluster 0 osdmap e284: 8 total, 8 up, 8 in 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[31149]: cluster 2024-03-07T03:17:11.959253+0000 mon.a (mon.0) 24 : cluster 0 Standby manager daemon x restarted 2024-03-07T03:17:12.128 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[31149]: cluster 2024-03-07T03:17:11.959379+0000 mon.a (mon.0) 25 : cluster 0 Standby manager daemon x started 2024-03-07T03:17:12.129 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[31149]: audit 2024-03-07T03:17:11.963586+0000 mon.b (mon.1) 200 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-07T03:17:12.129 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[31149]: audit 2024-03-07T03:17:11.964785+0000 mon.b (mon.1) 201 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-07T03:17:12.129 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[31149]: audit 2024-03-07T03:17:11.967700+0000 mon.b (mon.1) 202 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-07T03:17:12.129 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[31149]: audit 2024-03-07T03:17:11.969454+0000 mon.b (mon.1) 203 : audit 0 from='mgr.? 172.21.15.195:0/791865734' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-07T03:17:12.258 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:12.311 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[59535]: CherryPy Checker: 2024-03-07T03:17:12.311 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[59535]: The Application mounted at '' has an empty config. 2024-03-07T03:17:12.311 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[59535]: [07/Mar/2024:03:17:12] ENGINE Serving on http://:::9283 2024-03-07T03:17:12.312 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:17:12 smithi195 bash[59535]: [07/Mar/2024:03:17:12] ENGINE Bus STARTED 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.137+0000 7f6d0409e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: [07/Mar/2024:03:17:12] ENGINE Bus STARTING 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c65be3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c65be3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c65be3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c65be3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c65be3700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c613da700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c613da700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c613da700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c613da700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: debug 2024-03-07T03:17:12.365+0000 7f6c613da700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: CherryPy Checker: 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: The Application mounted at '' has an empty config. 2024-03-07T03:17:12.397 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: [07/Mar/2024:03:17:12] ENGINE Serving on http://:::9283 2024-03-07T03:17:12.754 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:12 smithi148 bash[72584]: [07/Mar/2024:03:17:12] ENGINE Bus STARTED 2024-03-07T03:17:13.290 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[72584]: debug 2024-03-07T03:17:13.185+0000 7f6cc280d700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-03-07T03:17:13.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: cluster 2024-03-07T03:17:12.030672+0000 mon.a (mon.0) 26 : cluster 0 mgrmap e33: y(active, since 65s), standbys: x 2024-03-07T03:17:13.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: cluster 2024-03-07T03:17:12.030750+0000 mon.a (mon.0) 27 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2024-03-07T03:17:13.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: cluster 2024-03-07T03:17:12.162432+0000 mon.a (mon.0) 28 : cluster 1 Active manager daemon y restarted 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: cluster 2024-03-07T03:17:12.163614+0000 mon.a (mon.0) 29 : cluster 1 Activating manager daemon y 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: cluster 2024-03-07T03:17:12.174227+0000 mon.a (mon.0) 30 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: cluster 2024-03-07T03:17:12.175212+0000 mon.a (mon.0) 31 : cluster 0 mgrmap e34: y(active, starting, since 0.0120619s), standbys: x 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.177047+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.177548+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.177786+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.179174+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.179404+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.179693+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.179840+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.180213+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.180478+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.180662+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.180855+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.181082+0000 mon.a (mon.0) 43 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:17:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.181365+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.181575+0000 mon.a (mon.0) 45 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: cluster 2024-03-07T03:17:12.030672+0000 mon.a (mon.0) 26 : cluster 0 mgrmap e33: y(active, since 65s), standbys: x 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: cluster 2024-03-07T03:17:12.030750+0000 mon.a (mon.0) 27 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: cluster 2024-03-07T03:17:12.162432+0000 mon.a (mon.0) 28 : cluster 1 Active manager daemon y restarted 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: cluster 2024-03-07T03:17:12.163614+0000 mon.a (mon.0) 29 : cluster 1 Activating manager daemon y 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: cluster 2024-03-07T03:17:12.174227+0000 mon.a (mon.0) 30 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: cluster 2024-03-07T03:17:12.175212+0000 mon.a (mon.0) 31 : cluster 0 mgrmap e34: y(active, starting, since 0.0120619s), standbys: x 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.177047+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:17:13.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.177548+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.177786+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.179174+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.179404+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.179693+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.179840+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.180213+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.180478+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.180662+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.180855+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.181082+0000 mon.a (mon.0) 43 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.181365+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.181575+0000 mon.a (mon.0) 45 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.182101+0000 mon.a (mon.0) 46 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.182345+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.182612+0000 mon.a (mon.0) 48 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:17:13.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.183130+0000 mon.a (mon.0) 49 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: cluster 2024-03-07T03:17:12.210161+0000 mon.a (mon.0) 50 : cluster 1 Manager daemon y is now available 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.260047+0000 mon.a (mon.0) 51 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.263284+0000 mon.a (mon.0) 52 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.297303+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.302843+0000 mon.a (mon.0) 54 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.309785+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[79819]: audit 2024-03-07T03:17:12.332290+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.182101+0000 mon.a (mon.0) 46 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.182345+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.182612+0000 mon.a (mon.0) 48 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:17:13.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.183130+0000 mon.a (mon.0) 49 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:17:13.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: cluster 2024-03-07T03:17:12.210161+0000 mon.a (mon.0) 50 : cluster 1 Manager daemon y is now available 2024-03-07T03:17:13.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.260047+0000 mon.a (mon.0) 51 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:13.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.263284+0000 mon.a (mon.0) 52 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:13.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.297303+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:13.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.302843+0000 mon.a (mon.0) 54 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:13.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.309785+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:17:13.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:13 smithi148 bash[31149]: audit 2024-03-07T03:17:12.332290+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:17:13.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: cluster 2024-03-07T03:17:12.030672+0000 mon.a (mon.0) 26 : cluster 0 mgrmap e33: y(active, since 65s), standbys: x 2024-03-07T03:17:13.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: cluster 2024-03-07T03:17:12.030750+0000 mon.a (mon.0) 27 : cluster 0 osdmap e285: 8 total, 8 up, 8 in 2024-03-07T03:17:13.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: cluster 2024-03-07T03:17:12.162432+0000 mon.a (mon.0) 28 : cluster 1 Active manager daemon y restarted 2024-03-07T03:17:13.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: cluster 2024-03-07T03:17:12.163614+0000 mon.a (mon.0) 29 : cluster 1 Activating manager daemon y 2024-03-07T03:17:13.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: cluster 2024-03-07T03:17:12.174227+0000 mon.a (mon.0) 30 : cluster 0 osdmap e286: 8 total, 8 up, 8 in 2024-03-07T03:17:13.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: cluster 2024-03-07T03:17:12.175212+0000 mon.a (mon.0) 31 : cluster 0 mgrmap e34: y(active, starting, since 0.0120619s), standbys: x 2024-03-07T03:17:13.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.177047+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.177548+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.177786+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.179174+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi148.qizikc"}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.179404+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi195.gtlmjn"}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.179693+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.179840+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.180213+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.180478+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.180662+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.180855+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.181082+0000 mon.a (mon.0) 43 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.181365+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.181575+0000 mon.a (mon.0) 45 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-07T03:17:13.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.182101+0000 mon.a (mon.0) 46 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.182345+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.182612+0000 mon.a (mon.0) 48 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.183130+0000 mon.a (mon.0) 49 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: cluster 2024-03-07T03:17:12.210161+0000 mon.a (mon.0) 50 : cluster 1 Manager daemon y is now available 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.260047+0000 mon.a (mon.0) 51 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.263284+0000 mon.a (mon.0) 52 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.297303+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.302843+0000 mon.a (mon.0) 54 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.309785+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-07T03:17:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[23221]: audit 2024-03-07T03:17:12.332290+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-07T03:17:13.548 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV1ParentAbort (4354 ms) 2024-03-07T03:17:13.548 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-03-07T03:17:13.603 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:13.626 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:13.644 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:13.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:13 smithi195 bash[59134]: ts=2024-03-07T03:17:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:14.242 INFO:teuthology.orchestra.run.smithi148.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:alertmanager.a smithi148 *:9093,9094 running (7m) 61s ago 8m 14.8M - 0.25.0 c8568f914cd2 a6ee726aca40 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:grafana.a smithi195 *:3000 running (7m) 30s ago 7m 79.6M - 9.4.7 954c08fa6188 a09a8cb7fec4 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi148.qizikc smithi148 running (5m) 61s ago 5m 13.2M - 18.2.1-718-gae75fc9d 0164bb8a7c67 d82cf82f7898 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi195.gtlmjn smithi195 running (5m) 30s ago 5m 15.8M - 18.2.1-718-gae75fc9d 0164bb8a7c67 f9ba5c67a194 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.x smithi195 *:8443,9283,8765 running (36s) 30s ago 15m 107M - 19.0.0-1905-gea9a5688 fdb83767c1ab 758d97044603 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.y smithi148 *:8443,9283,8765 running (84s) 61s ago 18m 470M - 19.0.0-1905-gea9a5688 fdb83767c1ab 85cf9ffcb609 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:mon.a smithi148 running (18m) 61s ago 18m 58.5M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 68e42a3c8dcc 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:mon.b smithi195 running (16m) 30s ago 16m 53.0M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 aa6d6fcf0602 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:mon.c smithi148 running (16m) 61s ago 16m 49.6M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 683a6360bfe4 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.a smithi148 *:9100 running (8m) 61s ago 8m 8435k - 1.5.0 0da6a335fe13 d9b895f99d5c 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.b smithi195 *:9100 running (8m) 30s ago 8m 9155k - 1.5.0 0da6a335fe13 be49ab11d18e 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:osd.0 smithi148 running (15m) 61s ago 15m 115M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 f8c58575ec57 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:osd.1 smithi148 running (14m) 61s ago 14m 104M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 3125450ae62f 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:osd.2 smithi148 running (13m) 61s ago 13m 87.6M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 b463a715bf36 2024-03-07T03:17:14.243 INFO:teuthology.orchestra.run.smithi148.stdout:osd.3 smithi148 running (12m) 61s ago 12m 102M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 180f02eefffa 2024-03-07T03:17:14.244 INFO:teuthology.orchestra.run.smithi148.stdout:osd.4 smithi195 running (12m) 30s ago 12m 111M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 5023ce93366a 2024-03-07T03:17:14.244 INFO:teuthology.orchestra.run.smithi148.stdout:osd.5 smithi195 running (11m) 30s ago 11m 83.0M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c850d7f8ae40 2024-03-07T03:17:14.244 INFO:teuthology.orchestra.run.smithi148.stdout:osd.6 smithi195 running (10m) 30s ago 10m 130M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 332f73f59fdc 2024-03-07T03:17:14.244 INFO:teuthology.orchestra.run.smithi148.stdout:osd.7 smithi195 running (9m) 30s ago 9m 103M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9a7e2329950b 2024-03-07T03:17:14.244 INFO:teuthology.orchestra.run.smithi148.stdout:prometheus.a smithi195 *:9095 running (43s) 30s ago 8m 36.2M - 2.43.0 a07b618ecd1d e5c6727b35a4 2024-03-07T03:17:14.454 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[79819]: cephadm 2024-03-07T03:17:12.759416+0000 mgr.y (mgr.34101) 1 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Bus STARTING 2024-03-07T03:17:14.454 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[79819]: cephadm 2024-03-07T03:17:12.879188+0000 mgr.y (mgr.34101) 2 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:17:14.454 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[79819]: cephadm 2024-03-07T03:17:12.980977+0000 mgr.y (mgr.34101) 3 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:17:14.454 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[79819]: cephadm 2024-03-07T03:17:12.981204+0000 mgr.y (mgr.34101) 4 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Bus STARTED 2024-03-07T03:17:14.454 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[79819]: cluster 2024-03-07T03:17:13.181540+0000 mon.a (mon.0) 57 : cluster 0 mgrmap e35: y(active, since 1.01838s), standbys: x 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[79819]: cluster 2024-03-07T03:17:13.200598+0000 mgr.y (mgr.34101) 5 : cluster 0 pgmap v3: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[79819]: cluster 2024-03-07T03:17:13.303744+0000 mon.a (mon.0) 58 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[31149]: cephadm 2024-03-07T03:17:12.759416+0000 mgr.y (mgr.34101) 1 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Bus STARTING 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[31149]: cephadm 2024-03-07T03:17:12.879188+0000 mgr.y (mgr.34101) 2 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[31149]: cephadm 2024-03-07T03:17:12.980977+0000 mgr.y (mgr.34101) 3 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[31149]: cephadm 2024-03-07T03:17:12.981204+0000 mgr.y (mgr.34101) 4 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Bus STARTED 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[31149]: cluster 2024-03-07T03:17:13.181540+0000 mon.a (mon.0) 57 : cluster 0 mgrmap e35: y(active, since 1.01838s), standbys: x 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[31149]: cluster 2024-03-07T03:17:13.200598+0000 mgr.y (mgr.34101) 5 : cluster 0 pgmap v3: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:14.455 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:14 smithi148 bash[31149]: cluster 2024-03-07T03:17:13.303744+0000 mon.a (mon.0) 58 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:14 smithi195 bash[23221]: cephadm 2024-03-07T03:17:12.759416+0000 mgr.y (mgr.34101) 1 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Bus STARTING 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:14 smithi195 bash[23221]: cephadm 2024-03-07T03:17:12.879188+0000 mgr.y (mgr.34101) 2 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Serving on https://172.21.15.148:7150 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:14 smithi195 bash[23221]: cephadm 2024-03-07T03:17:12.980977+0000 mgr.y (mgr.34101) 3 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Serving on http://172.21.15.148:8765 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:14 smithi195 bash[23221]: cephadm 2024-03-07T03:17:12.981204+0000 mgr.y (mgr.34101) 4 : cephadm 1 [07/Mar/2024:03:17:12] ENGINE Bus STARTED 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:14 smithi195 bash[23221]: cluster 2024-03-07T03:17:13.181540+0000 mon.a (mon.0) 57 : cluster 0 mgrmap e35: y(active, since 1.01838s), standbys: x 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:14 smithi195 bash[23221]: cluster 2024-03-07T03:17:13.200598+0000 mgr.y (mgr.34101) 5 : cluster 0 pgmap v3: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:14 smithi195 bash[23221]: cluster 2024-03-07T03:17:13.303744+0000 mon.a (mon.0) 58 : cluster 0 osdmap e287: 8 total, 8 up, 8 in 2024-03-07T03:17:14.978 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:17:15.326 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:15.481 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:15.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:15 smithi195 bash[23221]: cluster 2024-03-07T03:17:14.182116+0000 mgr.y (mgr.34101) 6 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:15.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:15 smithi195 bash[23221]: audit 2024-03-07T03:17:14.216400+0000 mgr.y (mgr.34101) 7 : audit 0 from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:15.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:15 smithi195 bash[23221]: cluster 2024-03-07T03:17:14.305575+0000 mon.a (mon.0) 59 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2024-03-07T03:17:15.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:15 smithi195 bash[23221]: cluster 2024-03-07T03:17:15.192276+0000 mon.a (mon.0) 60 : cluster 0 mgrmap e36: y(active, since 3s), standbys: x 2024-03-07T03:17:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[79819]: cluster 2024-03-07T03:17:14.182116+0000 mgr.y (mgr.34101) 6 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[79819]: audit 2024-03-07T03:17:14.216400+0000 mgr.y (mgr.34101) 7 : audit 0 from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[79819]: cluster 2024-03-07T03:17:14.305575+0000 mon.a (mon.0) 59 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2024-03-07T03:17:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[79819]: cluster 2024-03-07T03:17:15.192276+0000 mon.a (mon.0) 60 : cluster 0 mgrmap e36: y(active, since 3s), standbys: x 2024-03-07T03:17:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[31149]: cluster 2024-03-07T03:17:14.182116+0000 mgr.y (mgr.34101) 6 : cluster 0 pgmap v5: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[31149]: audit 2024-03-07T03:17:14.216400+0000 mgr.y (mgr.34101) 7 : audit 0 from='client.34140 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[31149]: cluster 2024-03-07T03:17:14.305575+0000 mon.a (mon.0) 59 : cluster 0 osdmap e288: 8 total, 8 up, 8 in 2024-03-07T03:17:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:15 smithi148 bash[31149]: cluster 2024-03-07T03:17:15.192276+0000 mon.a (mon.0) 60 : cluster 0 mgrmap e36: y(active, since 3s), standbys: x 2024-03-07T03:17:16.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:16 smithi195 bash[23221]: cluster 2024-03-07T03:17:15.308768+0000 mon.a (mon.0) 61 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2024-03-07T03:17:16.717 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:16 smithi148 bash[79819]: cluster 2024-03-07T03:17:15.308768+0000 mon.a (mon.0) 61 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2024-03-07T03:17:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:16 smithi148 bash[31149]: cluster 2024-03-07T03:17:15.308768+0000 mon.a (mon.0) 61 : cluster 0 osdmap e289: 8 total, 8 up, 8 in 2024-03-07T03:17:17.231 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV2ParentAbort (3683 ms) 2024-03-07T03:17:17.231 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-03-07T03:17:17.310 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:17.337 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:17.358 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:17.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:17:17] "GET /metrics HTTP/1.1" 200 36021 "" "Prometheus/2.43.0" 2024-03-07T03:17:17.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[79819]: cluster 2024-03-07T03:17:16.182750+0000 mgr.y (mgr.34101) 8 : cluster 0 pgmap v8: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:17.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[79819]: cluster 2024-03-07T03:17:16.337693+0000 mon.a (mon.0) 62 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-03-07T03:17:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[79819]: cluster 2024-03-07T03:17:16.337942+0000 mon.a (mon.0) 63 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2024-03-07T03:17:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[79819]: cluster 2024-03-07T03:17:17.005010+0000 mon.a (mon.0) 64 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2024-03-07T03:17:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[31149]: cluster 2024-03-07T03:17:16.182750+0000 mgr.y (mgr.34101) 8 : cluster 0 pgmap v8: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[31149]: cluster 2024-03-07T03:17:16.337693+0000 mon.a (mon.0) 62 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-03-07T03:17:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[31149]: cluster 2024-03-07T03:17:16.337942+0000 mon.a (mon.0) 63 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2024-03-07T03:17:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:17 smithi148 bash[31149]: cluster 2024-03-07T03:17:17.005010+0000 mon.a (mon.0) 64 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2024-03-07T03:17:17.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:17 smithi195 bash[23221]: cluster 2024-03-07T03:17:16.182750+0000 mgr.y (mgr.34101) 8 : cluster 0 pgmap v8: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:17.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:17 smithi195 bash[23221]: cluster 2024-03-07T03:17:16.337693+0000 mon.a (mon.0) 62 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-03-07T03:17:17.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:17 smithi195 bash[23221]: cluster 2024-03-07T03:17:16.337942+0000 mon.a (mon.0) 63 : cluster 0 osdmap e290: 8 total, 8 up, 8 in 2024-03-07T03:17:17.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:17 smithi195 bash[23221]: cluster 2024-03-07T03:17:17.005010+0000 mon.a (mon.0) 64 : cluster 0 osdmap e291: 8 total, 8 up, 8 in 2024-03-07T03:17:19.354 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:19.458 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:19.705 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:17:19.730 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[31149]: cluster 2024-03-07T03:17:18.183336+0000 mgr.y (mgr.34101) 9 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:19.730 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[31149]: cluster 2024-03-07T03:17:18.342387+0000 mon.a (mon.0) 65 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[31149]: audit 2024-03-07T03:17:18.447892+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[31149]: audit 2024-03-07T03:17:18.454770+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[31149]: audit 2024-03-07T03:17:18.733204+0000 mon.a (mon.0) 68 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[31149]: audit 2024-03-07T03:17:18.746652+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[79819]: cluster 2024-03-07T03:17:18.183336+0000 mgr.y (mgr.34101) 9 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[79819]: cluster 2024-03-07T03:17:18.342387+0000 mon.a (mon.0) 65 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[79819]: audit 2024-03-07T03:17:18.447892+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[79819]: audit 2024-03-07T03:17:18.454770+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[79819]: audit 2024-03-07T03:17:18.733204+0000 mon.a (mon.0) 68 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.731 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:19 smithi148 bash[79819]: audit 2024-03-07T03:17:18.746652+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:19 smithi195 bash[23221]: cluster 2024-03-07T03:17:18.183336+0000 mgr.y (mgr.34101) 9 : cluster 0 pgmap v11: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:19 smithi195 bash[23221]: cluster 2024-03-07T03:17:18.342387+0000 mon.a (mon.0) 65 : cluster 0 osdmap e292: 8 total, 8 up, 8 in 2024-03-07T03:17:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:19 smithi195 bash[23221]: audit 2024-03-07T03:17:18.447892+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:19 smithi195 bash[23221]: audit 2024-03-07T03:17:18.454770+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:19 smithi195 bash[23221]: audit 2024-03-07T03:17:18.733204+0000 mon.a (mon.0) 68 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:19 smithi195 bash[23221]: audit 2024-03-07T03:17:18.746652+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.738 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[31149]: cluster 2024-03-07T03:17:19.348507+0000 mon.a (mon.0) 70 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2024-03-07T03:17:20.738 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[31149]: audit 2024-03-07T03:17:19.509343+0000 mon.a (mon.0) 71 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.738 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[31149]: audit 2024-03-07T03:17:19.515024+0000 mon.a (mon.0) 72 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.738 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[31149]: audit 2024-03-07T03:17:19.814638+0000 mon.a (mon.0) 73 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.739 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[31149]: audit 2024-03-07T03:17:19.824693+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[79819]: cluster 2024-03-07T03:17:19.348507+0000 mon.a (mon.0) 70 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2024-03-07T03:17:20.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[79819]: audit 2024-03-07T03:17:19.509343+0000 mon.a (mon.0) 71 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[79819]: audit 2024-03-07T03:17:19.515024+0000 mon.a (mon.0) 72 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[79819]: audit 2024-03-07T03:17:19.814638+0000 mon.a (mon.0) 73 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:20 smithi148 bash[79819]: audit 2024-03-07T03:17:19.824693+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:20 smithi195 bash[23221]: cluster 2024-03-07T03:17:19.348507+0000 mon.a (mon.0) 70 : cluster 0 osdmap e293: 8 total, 8 up, 8 in 2024-03-07T03:17:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:20 smithi195 bash[23221]: audit 2024-03-07T03:17:19.509343+0000 mon.a (mon.0) 71 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:20 smithi195 bash[23221]: audit 2024-03-07T03:17:19.515024+0000 mon.a (mon.0) 72 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:20 smithi195 bash[23221]: audit 2024-03-07T03:17:19.814638+0000 mon.a (mon.0) 73 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:20 smithi195 bash[23221]: audit 2024-03-07T03:17:19.824693+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:20.857 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:21.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:21 smithi148 bash[79819]: cluster 2024-03-07T03:17:20.184126+0000 mgr.y (mgr.34101) 10 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 26 KiB/s wr, 407 op/s 2024-03-07T03:17:21.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:21 smithi148 bash[79819]: cluster 2024-03-07T03:17:20.517965+0000 mon.a (mon.0) 75 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2024-03-07T03:17:21.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:21 smithi148 bash[31149]: cluster 2024-03-07T03:17:20.184126+0000 mgr.y (mgr.34101) 10 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 26 KiB/s wr, 407 op/s 2024-03-07T03:17:21.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:21 smithi148 bash[31149]: cluster 2024-03-07T03:17:20.517965+0000 mon.a (mon.0) 75 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2024-03-07T03:17:21.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:21 smithi195 bash[23221]: cluster 2024-03-07T03:17:20.184126+0000 mgr.y (mgr.34101) 10 : cluster 0 pgmap v14: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 335 KiB/s rd, 26 KiB/s wr, 407 op/s 2024-03-07T03:17:21.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:21 smithi195 bash[23221]: cluster 2024-03-07T03:17:20.517965+0000 mon.a (mon.0) 75 : cluster 0 osdmap e294: 8 total, 8 up, 8 in 2024-03-07T03:17:22.112 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (4880 ms) 2024-03-07T03:17:22.112 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-03-07T03:17:22.180 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:22.207 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:22.230 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:22.561 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:22 smithi195 bash[59134]: ts=2024-03-07T03:17:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:22.791 INFO:teuthology.orchestra.run.smithi148.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:17:23.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[79819]: cluster 2024-03-07T03:17:21.838873+0000 mon.a (mon.0) 76 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2024-03-07T03:17:23.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[79819]: cluster 2024-03-07T03:17:22.008901+0000 mon.a (mon.0) 77 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2024-03-07T03:17:23.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[79819]: cluster 2024-03-07T03:17:22.184941+0000 mgr.y (mgr.34101) 11 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:23.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[79819]: audit 2024-03-07T03:17:22.805147+0000 mon.a (mon.0) 78 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:23.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[31149]: cluster 2024-03-07T03:17:21.838873+0000 mon.a (mon.0) 76 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2024-03-07T03:17:23.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[31149]: cluster 2024-03-07T03:17:22.008901+0000 mon.a (mon.0) 77 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2024-03-07T03:17:23.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[31149]: cluster 2024-03-07T03:17:22.184941+0000 mgr.y (mgr.34101) 11 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:23.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:22 smithi148 bash[31149]: audit 2024-03-07T03:17:22.805147+0000 mon.a (mon.0) 78 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:23.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:22 smithi195 bash[23221]: cluster 2024-03-07T03:17:21.838873+0000 mon.a (mon.0) 76 : cluster 0 osdmap e295: 8 total, 8 up, 8 in 2024-03-07T03:17:23.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:22 smithi195 bash[23221]: cluster 2024-03-07T03:17:22.008901+0000 mon.a (mon.0) 77 : cluster 0 osdmap e296: 8 total, 8 up, 8 in 2024-03-07T03:17:23.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:22 smithi195 bash[23221]: cluster 2024-03-07T03:17:22.184941+0000 mgr.y (mgr.34101) 11 : cluster 0 pgmap v18: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:23.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:22 smithi195 bash[23221]: audit 2024-03-07T03:17:22.805147+0000 mon.a (mon.0) 78 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:23.571 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'sleep 60' 2024-03-07T03:17:23.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:23 smithi195 bash[59134]: ts=2024-03-07T03:17:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:24.044 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:24.190 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:24.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:24 smithi148 bash[79819]: audit 2024-03-07T03:17:22.798726+0000 mgr.y (mgr.34101) 12 : audit 0 from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:24.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:24 smithi148 bash[79819]: cephadm 2024-03-07T03:17:22.799492+0000 mgr.y (mgr.34101) 13 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:17:24.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:24 smithi148 bash[79819]: cluster 2024-03-07T03:17:23.028203+0000 mon.a (mon.0) 79 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2024-03-07T03:17:24.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:24 smithi148 bash[31149]: audit 2024-03-07T03:17:22.798726+0000 mgr.y (mgr.34101) 12 : audit 0 from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:24.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:24 smithi148 bash[31149]: cephadm 2024-03-07T03:17:22.799492+0000 mgr.y (mgr.34101) 13 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:17:24.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:24 smithi148 bash[31149]: cluster 2024-03-07T03:17:23.028203+0000 mon.a (mon.0) 79 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2024-03-07T03:17:24.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:24 smithi195 bash[23221]: audit 2024-03-07T03:17:22.798726+0000 mgr.y (mgr.34101) 12 : audit 0 from='client.34152 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:17:24.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:24 smithi195 bash[23221]: cephadm 2024-03-07T03:17:22.799492+0000 mgr.y (mgr.34101) 13 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:17:24.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:24 smithi195 bash[23221]: cluster 2024-03-07T03:17:23.028203+0000 mon.a (mon.0) 79 : cluster 0 osdmap e297: 8 total, 8 up, 8 in 2024-03-07T03:17:25.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:25 smithi148 bash[79819]: cluster 2024-03-07T03:17:24.031305+0000 mon.a (mon.0) 80 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2024-03-07T03:17:25.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:25 smithi148 bash[79819]: cluster 2024-03-07T03:17:24.185752+0000 mgr.y (mgr.34101) 14 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 306 KiB/s rd, 19 KiB/s wr, 390 op/s 2024-03-07T03:17:25.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:25 smithi148 bash[31149]: cluster 2024-03-07T03:17:24.031305+0000 mon.a (mon.0) 80 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2024-03-07T03:17:25.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:25 smithi148 bash[31149]: cluster 2024-03-07T03:17:24.185752+0000 mgr.y (mgr.34101) 14 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 306 KiB/s rd, 19 KiB/s wr, 390 op/s 2024-03-07T03:17:25.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:25 smithi195 bash[23221]: cluster 2024-03-07T03:17:24.031305+0000 mon.a (mon.0) 80 : cluster 0 osdmap e298: 8 total, 8 up, 8 in 2024-03-07T03:17:25.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:25 smithi195 bash[23221]: cluster 2024-03-07T03:17:24.185752+0000 mgr.y (mgr.34101) 14 : cluster 0 pgmap v21: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 306 KiB/s rd, 19 KiB/s wr, 390 op/s 2024-03-07T03:17:25.497 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:26.276 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4164 ms) 2024-03-07T03:17:26.276 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-03-07T03:17:26.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:26 smithi195 bash[23221]: cluster 2024-03-07T03:17:25.052051+0000 mon.a (mon.0) 81 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2024-03-07T03:17:26.311 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:26.324 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:26.333 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:26.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:26 smithi148 bash[79819]: cluster 2024-03-07T03:17:25.052051+0000 mon.a (mon.0) 81 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2024-03-07T03:17:26.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:26 smithi148 bash[31149]: cluster 2024-03-07T03:17:25.052051+0000 mon.a (mon.0) 81 : cluster 0 osdmap e299: 8 total, 8 up, 8 in 2024-03-07T03:17:27.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:27 smithi195 bash[23221]: cluster 2024-03-07T03:17:26.051349+0000 mon.a (mon.0) 82 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2024-03-07T03:17:27.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:27 smithi195 bash[23221]: cluster 2024-03-07T03:17:26.186543+0000 mgr.y (mgr.34101) 15 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-07T03:17:27.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:27 smithi195 bash[23221]: cluster 2024-03-07T03:17:27.009727+0000 mon.a (mon.0) 83 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2024-03-07T03:17:27.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:17:27] "GET /metrics HTTP/1.1" 200 36021 "" "Prometheus/2.43.0" 2024-03-07T03:17:27.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:27 smithi148 bash[31149]: cluster 2024-03-07T03:17:26.051349+0000 mon.a (mon.0) 82 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2024-03-07T03:17:27.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:27 smithi148 bash[31149]: cluster 2024-03-07T03:17:26.186543+0000 mgr.y (mgr.34101) 15 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-07T03:17:27.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:27 smithi148 bash[31149]: cluster 2024-03-07T03:17:27.009727+0000 mon.a (mon.0) 83 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2024-03-07T03:17:27.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:27 smithi148 bash[79819]: cluster 2024-03-07T03:17:26.051349+0000 mon.a (mon.0) 82 : cluster 0 osdmap e300: 8 total, 8 up, 8 in 2024-03-07T03:17:27.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:27 smithi148 bash[79819]: cluster 2024-03-07T03:17:26.186543+0000 mgr.y (mgr.34101) 15 : cluster 0 pgmap v24: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 281 KiB/s rd, 17 KiB/s wr, 358 op/s 2024-03-07T03:17:27.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:27 smithi148 bash[79819]: cluster 2024-03-07T03:17:27.009727+0000 mon.a (mon.0) 83 : cluster 0 osdmap e301: 8 total, 8 up, 8 in 2024-03-07T03:17:28.020 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:28.156 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:28.308 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:17:28.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:28 smithi195 bash[23221]: audit 2024-03-07T03:17:27.293810+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:28.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:28 smithi195 bash[23221]: cluster 2024-03-07T03:17:28.011298+0000 mon.a (mon.0) 85 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2024-03-07T03:17:28.329 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:28 smithi148 bash[79819]: audit 2024-03-07T03:17:27.293810+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:28.329 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:28 smithi148 bash[79819]: cluster 2024-03-07T03:17:28.011298+0000 mon.a (mon.0) 85 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2024-03-07T03:17:28.329 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:28 smithi148 bash[31149]: audit 2024-03-07T03:17:27.293810+0000 mon.a (mon.0) 84 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:28.329 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:28 smithi148 bash[31149]: cluster 2024-03-07T03:17:28.011298+0000 mon.a (mon.0) 85 : cluster 0 osdmap e302: 8 total, 8 up, 8 in 2024-03-07T03:17:29.354 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:29 smithi148 bash[79819]: cluster 2024-03-07T03:17:28.187455+0000 mgr.y (mgr.34101) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:29.354 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:29 smithi148 bash[31149]: cluster 2024-03-07T03:17:28.187455+0000 mgr.y (mgr.34101) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:29.368 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:29 smithi195 bash[23221]: cluster 2024-03-07T03:17:28.187455+0000 mgr.y (mgr.34101) 16 : cluster 0 pgmap v27: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:17:29.402 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:30.303 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4027 ms) 2024-03-07T03:17:30.303 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-03-07T03:17:30.342 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:30.356 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:30.367 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:30.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:30 smithi148 bash[79819]: cluster 2024-03-07T03:17:29.087650+0000 mon.a (mon.0) 86 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2024-03-07T03:17:30.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:30 smithi148 bash[31149]: cluster 2024-03-07T03:17:29.087650+0000 mon.a (mon.0) 86 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2024-03-07T03:17:30.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:30 smithi195 bash[23221]: cluster 2024-03-07T03:17:29.087650+0000 mon.a (mon.0) 86 : cluster 0 osdmap e303: 8 total, 8 up, 8 in 2024-03-07T03:17:31.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: cluster 2024-03-07T03:17:30.099211+0000 mon.a (mon.0) 87 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2024-03-07T03:17:31.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.110558+0000 mon.a (mon.0) 88 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.125118+0000 mon.a (mon.0) 89 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.129210+0000 mon.a (mon.0) 90 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.133434+0000 mon.a (mon.0) 91 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.136274+0000 mgr.y (mgr.34101) 17 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.137081+0000 mgr.y (mgr.34101) 18 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: cluster 2024-03-07T03:17:30.188000+0000 mgr.y (mgr.34101) 19 : cluster 0 pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 19 KiB/s wr, 417 op/s 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.359337+0000 mgr.y (mgr.34101) 20 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.403456+0000 mgr.y (mgr.34101) 21 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.523845+0000 mgr.y (mgr.34101) 22 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.857477+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.862912+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.907486+0000 mon.a (mon.0) 94 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.916985+0000 mon.a (mon.0) 95 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:30.924880+0000 mon.a (mon.0) 96 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:31.002342+0000 mon.a (mon.0) 97 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:31.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:31.005204+0000 mon.a (mon.0) 98 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:31.007063+0000 mon.a (mon.0) 99 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[79819]: audit 2024-03-07T03:17:31.016107+0000 mon.a (mon.0) 100 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: cluster 2024-03-07T03:17:30.099211+0000 mon.a (mon.0) 87 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.110558+0000 mon.a (mon.0) 88 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.125118+0000 mon.a (mon.0) 89 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.129210+0000 mon.a (mon.0) 90 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.133434+0000 mon.a (mon.0) 91 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.136274+0000 mgr.y (mgr.34101) 17 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.137081+0000 mgr.y (mgr.34101) 18 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: cluster 2024-03-07T03:17:30.188000+0000 mgr.y (mgr.34101) 19 : cluster 0 pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 19 KiB/s wr, 417 op/s 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.359337+0000 mgr.y (mgr.34101) 20 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.403456+0000 mgr.y (mgr.34101) 21 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:17:31.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.523845+0000 mgr.y (mgr.34101) 22 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.857477+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.862912+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.907486+0000 mon.a (mon.0) 94 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.916985+0000 mon.a (mon.0) 95 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:30.924880+0000 mon.a (mon.0) 96 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:31.002342+0000 mon.a (mon.0) 97 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:31.005204+0000 mon.a (mon.0) 98 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:31.007063+0000 mon.a (mon.0) 99 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:31 smithi148 bash[31149]: audit 2024-03-07T03:17:31.016107+0000 mon.a (mon.0) 100 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: cluster 2024-03-07T03:17:30.099211+0000 mon.a (mon.0) 87 : cluster 0 osdmap e304: 8 total, 8 up, 8 in 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.110558+0000 mon.a (mon.0) 88 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.125118+0000 mon.a (mon.0) 89 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.129210+0000 mon.a (mon.0) 90 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.133434+0000 mon.a (mon.0) 91 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.136274+0000 mgr.y (mgr.34101) 17 : cephadm 1 Updating smithi148:/etc/ceph/ceph.conf 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.137081+0000 mgr.y (mgr.34101) 18 : cephadm 1 Updating smithi195:/etc/ceph/ceph.conf 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: cluster 2024-03-07T03:17:30.188000+0000 mgr.y (mgr.34101) 19 : cluster 0 pgmap v30: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 19 KiB/s wr, 417 op/s 2024-03-07T03:17:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.359337+0000 mgr.y (mgr.34101) 20 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.403456+0000 mgr.y (mgr.34101) 21 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.conf 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.523845+0000 mgr.y (mgr.34101) 22 : cephadm 1 Updating smithi195:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.857477+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.862912+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.907486+0000 mon.a (mon.0) 94 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.916985+0000 mon.a (mon.0) 95 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:30.924880+0000 mon.a (mon.0) 96 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:31.002342+0000 mon.a (mon.0) 97 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:31.005204+0000 mon.a (mon.0) 98 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:31.007063+0000 mon.a (mon.0) 99 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:17:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:31 smithi195 bash[23221]: audit 2024-03-07T03:17:31.016107+0000 mon.a (mon.0) 100 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:17:32.025 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:32.170 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:32.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.567868+0000 mgr.y (mgr.34101) 23 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:17:32.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.688352+0000 mgr.y (mgr.34101) 24 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:17:32.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[79819]: cephadm 2024-03-07T03:17:30.732538+0000 mgr.y (mgr.34101) 25 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:17:32.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[79819]: cluster 2024-03-07T03:17:31.135181+0000 mon.a (mon.0) 101 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2024-03-07T03:17:32.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[79819]: cluster 2024-03-07T03:17:32.011578+0000 mon.a (mon.0) 102 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2024-03-07T03:17:32.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.567868+0000 mgr.y (mgr.34101) 23 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:17:32.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.688352+0000 mgr.y (mgr.34101) 24 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:17:32.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[31149]: cephadm 2024-03-07T03:17:30.732538+0000 mgr.y (mgr.34101) 25 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:17:32.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[31149]: cluster 2024-03-07T03:17:31.135181+0000 mon.a (mon.0) 101 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2024-03-07T03:17:32.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:32 smithi148 bash[31149]: cluster 2024-03-07T03:17:32.011578+0000 mon.a (mon.0) 102 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2024-03-07T03:17:32.561 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:32 smithi195 bash[59134]: ts=2024-03-07T03:17:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:32 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.567868+0000 mgr.y (mgr.34101) 23 : cephadm 1 Updating smithi148:/etc/ceph/ceph.client.admin.keyring 2024-03-07T03:17:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:32 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.688352+0000 mgr.y (mgr.34101) 24 : cephadm 1 Updating smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:17:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:32 smithi195 bash[23221]: cephadm 2024-03-07T03:17:30.732538+0000 mgr.y (mgr.34101) 25 : cephadm 1 Updating smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/config/ceph.client.admin.keyring 2024-03-07T03:17:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:32 smithi195 bash[23221]: cluster 2024-03-07T03:17:31.135181+0000 mon.a (mon.0) 101 : cluster 0 osdmap e305: 8 total, 8 up, 8 in 2024-03-07T03:17:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:32 smithi195 bash[23221]: cluster 2024-03-07T03:17:32.011578+0000 mon.a (mon.0) 102 : cluster 0 osdmap e306: 8 total, 8 up, 8 in 2024-03-07T03:17:33.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:33 smithi195 bash[23221]: cluster 2024-03-07T03:17:32.188770+0000 mgr.y (mgr.34101) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 19 KiB/s wr, 417 op/s 2024-03-07T03:17:33.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:33 smithi148 bash[79819]: cluster 2024-03-07T03:17:32.188770+0000 mgr.y (mgr.34101) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 19 KiB/s wr, 417 op/s 2024-03-07T03:17:33.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:33 smithi148 bash[31149]: cluster 2024-03-07T03:17:32.188770+0000 mgr.y (mgr.34101) 26 : cluster 0 pgmap v33: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 19 KiB/s wr, 417 op/s 2024-03-07T03:17:33.552 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:33.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:33 smithi195 bash[59134]: ts=2024-03-07T03:17:33.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:34.428 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (4126 ms) 2024-03-07T03:17:34.429 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-03-07T03:17:34.514 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:34.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:34 smithi148 bash[79819]: cluster 2024-03-07T03:17:33.152673+0000 mon.a (mon.0) 103 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2024-03-07T03:17:34.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:34 smithi148 bash[31149]: cluster 2024-03-07T03:17:33.152673+0000 mon.a (mon.0) 103 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2024-03-07T03:17:34.544 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:34.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:34 smithi195 bash[23221]: cluster 2024-03-07T03:17:33.152673+0000 mon.a (mon.0) 103 : cluster 0 osdmap e307: 8 total, 8 up, 8 in 2024-03-07T03:17:34.567 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:35 smithi148 bash[79819]: cluster 2024-03-07T03:17:34.151297+0000 mon.a (mon.0) 104 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2024-03-07T03:17:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:35 smithi148 bash[79819]: cluster 2024-03-07T03:17:34.189823+0000 mgr.y (mgr.34101) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 297 KiB/s rd, 22 KiB/s wr, 384 op/s 2024-03-07T03:17:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:35 smithi148 bash[31149]: cluster 2024-03-07T03:17:34.151297+0000 mon.a (mon.0) 104 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2024-03-07T03:17:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:35 smithi148 bash[31149]: cluster 2024-03-07T03:17:34.189823+0000 mgr.y (mgr.34101) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 297 KiB/s rd, 22 KiB/s wr, 384 op/s 2024-03-07T03:17:35.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:35 smithi195 bash[23221]: cluster 2024-03-07T03:17:34.151297+0000 mon.a (mon.0) 104 : cluster 0 osdmap e308: 8 total, 8 up, 8 in 2024-03-07T03:17:35.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:35 smithi195 bash[23221]: cluster 2024-03-07T03:17:34.189823+0000 mgr.y (mgr.34101) 27 : cluster 0 pgmap v36: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 297 KiB/s rd, 22 KiB/s wr, 384 op/s 2024-03-07T03:17:36.198 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:36.352 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:36.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:36 smithi148 bash[79819]: cluster 2024-03-07T03:17:35.174344+0000 mon.a (mon.0) 105 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2024-03-07T03:17:36.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:36 smithi148 bash[31149]: cluster 2024-03-07T03:17:35.174344+0000 mon.a (mon.0) 105 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2024-03-07T03:17:36.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:36 smithi195 bash[23221]: cluster 2024-03-07T03:17:35.174344+0000 mon.a (mon.0) 105 : cluster 0 osdmap e309: 8 total, 8 up, 8 in 2024-03-07T03:17:37.440 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:37.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:17:37] "GET /metrics HTTP/1.1" 200 38839 "" "Prometheus/2.43.0" 2024-03-07T03:17:37.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:37 smithi148 bash[79819]: cluster 2024-03-07T03:17:36.174844+0000 mon.a (mon.0) 106 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2024-03-07T03:17:37.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:37 smithi148 bash[79819]: cluster 2024-03-07T03:17:36.194545+0000 mgr.y (mgr.34101) 28 : cluster 0 pgmap v39: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 297 KiB/s rd, 22 KiB/s wr, 384 op/s 2024-03-07T03:17:37.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:37 smithi148 bash[79819]: cluster 2024-03-07T03:17:37.016627+0000 mon.a (mon.0) 107 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2024-03-07T03:17:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:37 smithi148 bash[31149]: cluster 2024-03-07T03:17:36.174844+0000 mon.a (mon.0) 106 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2024-03-07T03:17:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:37 smithi148 bash[31149]: cluster 2024-03-07T03:17:36.194545+0000 mgr.y (mgr.34101) 28 : cluster 0 pgmap v39: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 297 KiB/s rd, 22 KiB/s wr, 384 op/s 2024-03-07T03:17:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:37 smithi148 bash[31149]: cluster 2024-03-07T03:17:37.016627+0000 mon.a (mon.0) 107 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2024-03-07T03:17:37.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:37 smithi195 bash[23221]: cluster 2024-03-07T03:17:36.174844+0000 mon.a (mon.0) 106 : cluster 0 osdmap e310: 8 total, 8 up, 8 in 2024-03-07T03:17:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:37 smithi195 bash[23221]: cluster 2024-03-07T03:17:36.194545+0000 mgr.y (mgr.34101) 28 : cluster 0 pgmap v39: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 297 KiB/s rd, 22 KiB/s wr, 384 op/s 2024-03-07T03:17:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:37 smithi195 bash[23221]: cluster 2024-03-07T03:17:37.016627+0000 mon.a (mon.0) 107 : cluster 0 osdmap e311: 8 total, 8 up, 8 in 2024-03-07T03:17:38.476 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4047 ms) 2024-03-07T03:17:38.476 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-03-07T03:17:38.561 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:38.591 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 0~10 2024-03-07T03:17:38.614 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:39.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:39 smithi148 bash[79819]: cluster 2024-03-07T03:17:38.195731+0000 mgr.y (mgr.34101) 29 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.2 KiB/s wr, 34 op/s 2024-03-07T03:17:39.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:39 smithi148 bash[79819]: cluster 2024-03-07T03:17:38.202788+0000 mon.a (mon.0) 108 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2024-03-07T03:17:39.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:39 smithi148 bash[31149]: cluster 2024-03-07T03:17:38.195731+0000 mgr.y (mgr.34101) 29 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.2 KiB/s wr, 34 op/s 2024-03-07T03:17:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:39 smithi148 bash[31149]: cluster 2024-03-07T03:17:38.202788+0000 mon.a (mon.0) 108 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2024-03-07T03:17:39.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:39 smithi195 bash[23221]: cluster 2024-03-07T03:17:38.195731+0000 mgr.y (mgr.34101) 29 : cluster 0 pgmap v41: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 21 KiB/s rd, 2.2 KiB/s wr, 34 op/s 2024-03-07T03:17:39.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:39 smithi195 bash[23221]: cluster 2024-03-07T03:17:38.202788+0000 mon.a (mon.0) 108 : cluster 0 osdmap e312: 8 total, 8 up, 8 in 2024-03-07T03:17:40.238 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:40.393 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:17:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:40 smithi148 bash[79819]: cluster 2024-03-07T03:17:39.211526+0000 mon.a (mon.0) 109 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2024-03-07T03:17:40.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:40 smithi148 bash[31149]: cluster 2024-03-07T03:17:39.211526+0000 mon.a (mon.0) 109 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2024-03-07T03:17:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:40 smithi195 bash[23221]: cluster 2024-03-07T03:17:39.211526+0000 mon.a (mon.0) 109 : cluster 0 osdmap e313: 8 total, 8 up, 8 in 2024-03-07T03:17:41.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:41 smithi148 bash[79819]: cluster 2024-03-07T03:17:40.196576+0000 mgr.y (mgr.34101) 30 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 348 KiB/s rd, 25 KiB/s wr, 443 op/s 2024-03-07T03:17:41.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:41 smithi148 bash[79819]: cluster 2024-03-07T03:17:40.222336+0000 mon.a (mon.0) 110 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2024-03-07T03:17:41.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:41 smithi148 bash[31149]: cluster 2024-03-07T03:17:40.196576+0000 mgr.y (mgr.34101) 30 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 348 KiB/s rd, 25 KiB/s wr, 443 op/s 2024-03-07T03:17:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:41 smithi148 bash[31149]: cluster 2024-03-07T03:17:40.222336+0000 mon.a (mon.0) 110 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2024-03-07T03:17:41.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:41 smithi195 bash[23221]: cluster 2024-03-07T03:17:40.196576+0000 mgr.y (mgr.34101) 30 : cluster 0 pgmap v44: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 348 KiB/s rd, 25 KiB/s wr, 443 op/s 2024-03-07T03:17:41.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:41 smithi195 bash[23221]: cluster 2024-03-07T03:17:40.222336+0000 mon.a (mon.0) 110 : cluster 0 osdmap e314: 8 total, 8 up, 8 in 2024-03-07T03:17:41.660 INFO:tasks.workunit.client.0.smithi148.stdout:migration_abort 2024-03-07T03:17:42.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (3812 ms) 2024-03-07T03:17:42.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-03-07T03:17:42.384 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:42.412 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 66829241~1357195 2024-03-07T03:17:42.456 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 53620820~6925238 2024-03-07T03:17:42.456 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 79798998~1275309 2024-03-07T03:17:42.483 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 17715513~3224153 2024-03-07T03:17:42.483 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 58917149~5686931 2024-03-07T03:17:42.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:42 smithi148 bash[79819]: cluster 2024-03-07T03:17:41.234726+0000 mon.a (mon.0) 111 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2024-03-07T03:17:42.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:42 smithi148 bash[79819]: cluster 2024-03-07T03:17:42.017949+0000 mon.a (mon.0) 112 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2024-03-07T03:17:42.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:42 smithi148 bash[31149]: cluster 2024-03-07T03:17:41.234726+0000 mon.a (mon.0) 111 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2024-03-07T03:17:42.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:42 smithi148 bash[31149]: cluster 2024-03-07T03:17:42.017949+0000 mon.a (mon.0) 112 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2024-03-07T03:17:42.553 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 57176003~1716640 2024-03-07T03:17:42.559 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 50070752~2531014 2024-03-07T03:17:42.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:42 smithi195 bash[23221]: cluster 2024-03-07T03:17:41.234726+0000 mon.a (mon.0) 111 : cluster 0 osdmap e315: 8 total, 8 up, 8 in 2024-03-07T03:17:42.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:42 smithi195 bash[23221]: cluster 2024-03-07T03:17:42.017949+0000 mon.a (mon.0) 112 : cluster 0 osdmap e316: 8 total, 8 up, 8 in 2024-03-07T03:17:42.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:42 smithi195 bash[59134]: ts=2024-03-07T03:17:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:42.599 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 53551377~4952770 2024-03-07T03:17:42.605 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap0 2024-03-07T03:17:43.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:43 smithi195 bash[23221]: cluster 2024-03-07T03:17:42.197353+0000 mgr.y (mgr.34101) 31 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 166 KiB/s rd, 14 KiB/s wr, 208 op/s 2024-03-07T03:17:43.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:43 smithi195 bash[23221]: audit 2024-03-07T03:17:42.294148+0000 mon.a (mon.0) 113 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:43.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:43 smithi148 bash[79819]: cluster 2024-03-07T03:17:42.197353+0000 mgr.y (mgr.34101) 31 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 166 KiB/s rd, 14 KiB/s wr, 208 op/s 2024-03-07T03:17:43.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:43 smithi148 bash[79819]: audit 2024-03-07T03:17:42.294148+0000 mon.a (mon.0) 113 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:43.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:43 smithi148 bash[31149]: cluster 2024-03-07T03:17:42.197353+0000 mgr.y (mgr.34101) 31 : cluster 0 pgmap v48: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 166 KiB/s rd, 14 KiB/s wr, 208 op/s 2024-03-07T03:17:43.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:43 smithi148 bash[31149]: audit 2024-03-07T03:17:42.294148+0000 mon.a (mon.0) 113 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:43.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:43 smithi195 bash[59134]: ts=2024-03-07T03:17:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:44.272 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap0 2024-03-07T03:17:44.303 INFO:tasks.workunit.client.0.smithi148.stdout:clone image159 -> image162 2024-03-07T03:17:44.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:44 smithi148 bash[79819]: cluster 2024-03-07T03:17:43.241725+0000 mon.a (mon.0) 114 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2024-03-07T03:17:44.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:44 smithi148 bash[31149]: cluster 2024-03-07T03:17:43.241725+0000 mon.a (mon.0) 114 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2024-03-07T03:17:44.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:44 smithi195 bash[23221]: cluster 2024-03-07T03:17:43.241725+0000 mon.a (mon.0) 114 : cluster 0 osdmap e317: 8 total, 8 up, 8 in 2024-03-07T03:17:44.564 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 24706320~4882362 2024-03-07T03:17:44.665 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 40974856~6813265 2024-03-07T03:17:44.688 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 42854812~4244887 2024-03-07T03:17:44.732 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 40794533~5517353 2024-03-07T03:17:44.739 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 45438045~1031842 2024-03-07T03:17:44.751 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 66367826~7368006 2024-03-07T03:17:44.794 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 33891026~2054124 2024-03-07T03:17:44.828 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 590754~7367947 2024-03-07T03:17:44.844 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:45.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:45 smithi148 bash[79819]: cluster 2024-03-07T03:17:44.198183+0000 mgr.y (mgr.34101) 32 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 935 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 5.2 MiB/s wr, 411 op/s 2024-03-07T03:17:45.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:45 smithi148 bash[79819]: cluster 2024-03-07T03:17:44.258399+0000 mon.a (mon.0) 115 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2024-03-07T03:17:45.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:45 smithi148 bash[31149]: cluster 2024-03-07T03:17:44.198183+0000 mgr.y (mgr.34101) 32 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 935 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 5.2 MiB/s wr, 411 op/s 2024-03-07T03:17:45.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:45 smithi148 bash[31149]: cluster 2024-03-07T03:17:44.258399+0000 mon.a (mon.0) 115 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2024-03-07T03:17:45.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:45 smithi195 bash[23221]: cluster 2024-03-07T03:17:44.198183+0000 mgr.y (mgr.34101) 32 : cluster 0 pgmap v50: 137 pgs: 137 active+clean; 935 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 5.2 MiB/s wr, 411 op/s 2024-03-07T03:17:45.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:45 smithi195 bash[23221]: cluster 2024-03-07T03:17:44.258399+0000 mon.a (mon.0) 115 : cluster 0 osdmap e318: 8 total, 8 up, 8 in 2024-03-07T03:17:46.293 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:17:46.326 INFO:tasks.workunit.client.0.smithi148.stdout:clone image162 -> image164 2024-03-07T03:17:46.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:46 smithi148 bash[79819]: cluster 2024-03-07T03:17:45.272096+0000 mon.a (mon.0) 116 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2024-03-07T03:17:46.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:46 smithi148 bash[31149]: cluster 2024-03-07T03:17:45.272096+0000 mon.a (mon.0) 116 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2024-03-07T03:17:46.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:46 smithi195 bash[23221]: cluster 2024-03-07T03:17:45.272096+0000 mon.a (mon.0) 116 : cluster 0 osdmap e319: 8 total, 8 up, 8 in 2024-03-07T03:17:46.650 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 78993800~4892280 2024-03-07T03:17:46.732 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 44133644~1192946 2024-03-07T03:17:46.756 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 76839267~7046813 2024-03-07T03:17:46.816 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 76686691~646351 2024-03-07T03:17:46.822 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 81096594~2648690 2024-03-07T03:17:46.848 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 72970051~4408949 2024-03-07T03:17:46.863 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 26068527~4008206 2024-03-07T03:17:46.953 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 27598687~5401684 2024-03-07T03:17:46.960 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:17:47.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:17:47] "GET /metrics HTTP/1.1" 200 38835 "" "Prometheus/2.43.0" 2024-03-07T03:17:47.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:47 smithi148 bash[79819]: cluster 2024-03-07T03:17:46.198918+0000 mgr.y (mgr.34101) 33 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 935 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 MiB/s wr, 393 op/s 2024-03-07T03:17:47.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:47 smithi148 bash[79819]: cluster 2024-03-07T03:17:46.281713+0000 mon.a (mon.0) 117 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2024-03-07T03:17:47.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:47 smithi148 bash[79819]: cluster 2024-03-07T03:17:47.019907+0000 mon.a (mon.0) 118 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2024-03-07T03:17:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:47 smithi148 bash[31149]: cluster 2024-03-07T03:17:46.198918+0000 mgr.y (mgr.34101) 33 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 935 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 MiB/s wr, 393 op/s 2024-03-07T03:17:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:47 smithi148 bash[31149]: cluster 2024-03-07T03:17:46.281713+0000 mon.a (mon.0) 117 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2024-03-07T03:17:47.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:47 smithi148 bash[31149]: cluster 2024-03-07T03:17:47.019907+0000 mon.a (mon.0) 118 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2024-03-07T03:17:47.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:47 smithi195 bash[23221]: cluster 2024-03-07T03:17:46.198918+0000 mgr.y (mgr.34101) 33 : cluster 0 pgmap v53: 137 pgs: 137 active+clean; 935 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 290 KiB/s rd, 5.0 MiB/s wr, 393 op/s 2024-03-07T03:17:47.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:47 smithi195 bash[23221]: cluster 2024-03-07T03:17:46.281713+0000 mon.a (mon.0) 117 : cluster 0 osdmap e320: 8 total, 8 up, 8 in 2024-03-07T03:17:47.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:47 smithi195 bash[23221]: cluster 2024-03-07T03:17:47.019907+0000 mon.a (mon.0) 118 : cluster 0 osdmap e321: 8 total, 8 up, 8 in 2024-03-07T03:17:48.035 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap2 2024-03-07T03:17:48.063 INFO:tasks.workunit.client.0.smithi148.stdout:clone image164 -> image166 2024-03-07T03:17:48.453 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 104007931 2024-03-07T03:17:48.481 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 48795807~5451872 2024-03-07T03:17:48.593 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 99631453~4376478 2024-03-07T03:17:48.593 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 5442860~7477463 2024-03-07T03:17:48.659 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 12912071~2612394 2024-03-07T03:17:48.665 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 30230433~2614780 2024-03-07T03:17:48.706 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 65096442~3051215 2024-03-07T03:17:48.754 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 18343784~8095429 2024-03-07T03:17:48.825 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 67415033~4062374 2024-03-07T03:17:48.839 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:17:49.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:49 smithi148 bash[79819]: cluster 2024-03-07T03:17:48.026512+0000 mon.a (mon.0) 119 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2024-03-07T03:17:49.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:49 smithi148 bash[79819]: cluster 2024-03-07T03:17:48.200477+0000 mgr.y (mgr.34101) 34 : cluster 0 pgmap v57: 137 pgs: 137 active+clean; 989 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 13 MiB/s wr, 128 op/s 2024-03-07T03:17:49.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:49 smithi148 bash[31149]: cluster 2024-03-07T03:17:48.026512+0000 mon.a (mon.0) 119 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2024-03-07T03:17:49.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:49 smithi148 bash[31149]: cluster 2024-03-07T03:17:48.200477+0000 mgr.y (mgr.34101) 34 : cluster 0 pgmap v57: 137 pgs: 137 active+clean; 989 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 13 MiB/s wr, 128 op/s 2024-03-07T03:17:49.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:49 smithi195 bash[23221]: cluster 2024-03-07T03:17:48.026512+0000 mon.a (mon.0) 119 : cluster 0 osdmap e322: 8 total, 8 up, 8 in 2024-03-07T03:17:49.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:49 smithi195 bash[23221]: cluster 2024-03-07T03:17:48.200477+0000 mgr.y (mgr.34101) 34 : cluster 0 pgmap v57: 137 pgs: 137 active+clean; 989 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 13 MiB/s wr, 128 op/s 2024-03-07T03:17:50.061 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap3 2024-03-07T03:17:50.091 INFO:tasks.workunit.client.0.smithi148.stdout:clone image166 -> image168 2024-03-07T03:17:50.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:50 smithi148 bash[79819]: cluster 2024-03-07T03:17:49.038368+0000 mon.a (mon.0) 120 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2024-03-07T03:17:50.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:50 smithi148 bash[31149]: cluster 2024-03-07T03:17:49.038368+0000 mon.a (mon.0) 120 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2024-03-07T03:17:50.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:50 smithi195 bash[23221]: cluster 2024-03-07T03:17:49.038368+0000 mon.a (mon.0) 120 : cluster 0 osdmap e323: 8 total, 8 up, 8 in 2024-03-07T03:17:50.526 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 145312235 2024-03-07T03:17:50.898 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.StressNoMigrate (8610 ms) 2024-03-07T03:17:50.898 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.Stress 2024-03-07T03:17:50.986 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:17:51.016 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 18009259~1359952 2024-03-07T03:17:51.055 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 30934725~8142898 2024-03-07T03:17:51.055 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 20539713~7093231 2024-03-07T03:17:51.124 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 54437955~6671604 2024-03-07T03:17:51.124 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 23383340~6195360 2024-03-07T03:17:51.174 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 17928604~5364270 2024-03-07T03:17:51.181 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 64193982~3669388 2024-03-07T03:17:51.225 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 62947914~5527866 2024-03-07T03:17:51.232 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap0 2024-03-07T03:17:51.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:51 smithi195 bash[23221]: cluster 2024-03-07T03:17:50.050068+0000 mon.a (mon.0) 121 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2024-03-07T03:17:51.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:51 smithi195 bash[23221]: cluster 2024-03-07T03:17:50.200917+0000 mgr.y (mgr.34101) 35 : cluster 0 pgmap v60: 137 pgs: 137 active+clean; 1018 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 17 MiB/s wr, 458 op/s 2024-03-07T03:17:51.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:51 smithi148 bash[79819]: cluster 2024-03-07T03:17:50.050068+0000 mon.a (mon.0) 121 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2024-03-07T03:17:51.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:51 smithi148 bash[79819]: cluster 2024-03-07T03:17:50.200917+0000 mgr.y (mgr.34101) 35 : cluster 0 pgmap v60: 137 pgs: 137 active+clean; 1018 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 17 MiB/s wr, 458 op/s 2024-03-07T03:17:51.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:51 smithi148 bash[31149]: cluster 2024-03-07T03:17:50.050068+0000 mon.a (mon.0) 121 : cluster 0 osdmap e324: 8 total, 8 up, 8 in 2024-03-07T03:17:51.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:51 smithi148 bash[31149]: cluster 2024-03-07T03:17:50.200917+0000 mgr.y (mgr.34101) 35 : cluster 0 pgmap v60: 137 pgs: 137 active+clean; 1018 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.9 MiB/s rd, 17 MiB/s wr, 458 op/s 2024-03-07T03:17:52.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:52 smithi195 bash[59134]: ts=2024-03-07T03:17:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:53.075 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap0 2024-03-07T03:17:53.110 INFO:tasks.workunit.client.0.smithi148.stdout:clone image169 -> image172 2024-03-07T03:17:53.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:53 smithi195 bash[23221]: cluster 2024-03-07T03:17:52.064022+0000 mon.a (mon.0) 122 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2024-03-07T03:17:53.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:53 smithi195 bash[23221]: cluster 2024-03-07T03:17:52.201650+0000 mgr.y (mgr.34101) 36 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 1018 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 16 MiB/s wr, 430 op/s 2024-03-07T03:17:53.354 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 45931070~6840052 2024-03-07T03:17:53.447 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1654592~3389478 2024-03-07T03:17:53.462 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 14647515~7039246 2024-03-07T03:17:53.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:53 smithi148 bash[79819]: cluster 2024-03-07T03:17:52.064022+0000 mon.a (mon.0) 122 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2024-03-07T03:17:53.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:53 smithi148 bash[79819]: cluster 2024-03-07T03:17:52.201650+0000 mgr.y (mgr.34101) 36 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 1018 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 16 MiB/s wr, 430 op/s 2024-03-07T03:17:53.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:53 smithi148 bash[31149]: cluster 2024-03-07T03:17:52.064022+0000 mon.a (mon.0) 122 : cluster 0 osdmap e325: 8 total, 8 up, 8 in 2024-03-07T03:17:53.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:53 smithi148 bash[31149]: cluster 2024-03-07T03:17:52.201650+0000 mgr.y (mgr.34101) 36 : cluster 0 pgmap v62: 137 pgs: 137 active+clean; 1018 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 3.6 MiB/s rd, 16 MiB/s wr, 430 op/s 2024-03-07T03:17:53.541 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 33717597~1361413 2024-03-07T03:17:53.552 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 78756310~1932673 2024-03-07T03:17:53.581 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 67624890~333898 2024-03-07T03:17:53.595 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 26859675~5802615 2024-03-07T03:17:53.679 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 30722729~1748070 2024-03-07T03:17:53.685 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:17:53.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:17:53 smithi195 bash[59134]: ts=2024-03-07T03:17:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:17:54.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:54 smithi195 bash[23221]: cluster 2024-03-07T03:17:53.066220+0000 mon.a (mon.0) 123 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2024-03-07T03:17:54.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:54 smithi148 bash[79819]: cluster 2024-03-07T03:17:53.066220+0000 mon.a (mon.0) 123 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2024-03-07T03:17:54.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:54 smithi148 bash[31149]: cluster 2024-03-07T03:17:53.066220+0000 mon.a (mon.0) 123 : cluster 0 osdmap e326: 8 total, 8 up, 8 in 2024-03-07T03:17:55.102 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 135148522 2024-03-07T03:17:55.128 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 9224432~5745749 2024-03-07T03:17:55.145 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 32863727~7126155 2024-03-07T03:17:55.145 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 44635819~1534662 2024-03-07T03:17:55.146 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 37781900~7632653 2024-03-07T03:17:55.196 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 53803351~5416801 2024-03-07T03:17:55.200 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 46284413~3182362 2024-03-07T03:17:55.201 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 65902110~1923147 2024-03-07T03:17:55.202 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 24101056~1192370 2024-03-07T03:17:55.203 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:17:55.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:55 smithi148 bash[79819]: cluster 2024-03-07T03:17:54.083745+0000 mon.a (mon.0) 124 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2024-03-07T03:17:55.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:55 smithi148 bash[79819]: cluster 2024-03-07T03:17:54.202438+0000 mgr.y (mgr.34101) 37 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 22 MiB/s wr, 528 op/s 2024-03-07T03:17:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:55 smithi148 bash[31149]: cluster 2024-03-07T03:17:54.083745+0000 mon.a (mon.0) 124 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2024-03-07T03:17:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:55 smithi148 bash[31149]: cluster 2024-03-07T03:17:54.202438+0000 mgr.y (mgr.34101) 37 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 22 MiB/s wr, 528 op/s 2024-03-07T03:17:55.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:55 smithi195 bash[23221]: cluster 2024-03-07T03:17:54.083745+0000 mon.a (mon.0) 124 : cluster 0 osdmap e327: 8 total, 8 up, 8 in 2024-03-07T03:17:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:55 smithi195 bash[23221]: cluster 2024-03-07T03:17:54.202438+0000 mgr.y (mgr.34101) 37 : cluster 0 pgmap v65: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 22 MiB/s wr, 528 op/s 2024-03-07T03:17:56.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:56 smithi148 bash[79819]: cluster 2024-03-07T03:17:55.086336+0000 mon.a (mon.0) 125 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2024-03-07T03:17:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:56 smithi148 bash[31149]: cluster 2024-03-07T03:17:55.086336+0000 mon.a (mon.0) 125 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2024-03-07T03:17:56.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:56 smithi195 bash[23221]: cluster 2024-03-07T03:17:55.086336+0000 mon.a (mon.0) 125 : cluster 0 osdmap e328: 8 total, 8 up, 8 in 2024-03-07T03:17:57.132 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap2 2024-03-07T03:17:57.158 INFO:tasks.workunit.client.0.smithi148.stdout:clone image172 -> image174 2024-03-07T03:17:57.485 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 122379665 2024-03-07T03:17:57.529 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 60185657~5984196 2024-03-07T03:17:57.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:57 smithi148 bash[79819]: cluster 2024-03-07T03:17:56.109211+0000 mon.a (mon.0) 126 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2024-03-07T03:17:57.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:57 smithi148 bash[79819]: cluster 2024-03-07T03:17:56.203244+0000 mgr.y (mgr.34101) 38 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 23 MiB/s wr, 549 op/s 2024-03-07T03:17:57.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:17:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:17:57] "GET /metrics HTTP/1.1" 200 38835 "" "Prometheus/2.43.0" 2024-03-07T03:17:57.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:57 smithi148 bash[31149]: cluster 2024-03-07T03:17:56.109211+0000 mon.a (mon.0) 126 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2024-03-07T03:17:57.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:57 smithi148 bash[31149]: cluster 2024-03-07T03:17:56.203244+0000 mgr.y (mgr.34101) 38 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 23 MiB/s wr, 549 op/s 2024-03-07T03:17:57.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:57 smithi195 bash[23221]: cluster 2024-03-07T03:17:56.109211+0000 mon.a (mon.0) 126 : cluster 0 osdmap e329: 8 total, 8 up, 8 in 2024-03-07T03:17:57.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:57 smithi195 bash[23221]: cluster 2024-03-07T03:17:56.203244+0000 mgr.y (mgr.34101) 38 : cluster 0 pgmap v68: 137 pgs: 137 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 24 MiB/s rd, 23 MiB/s wr, 549 op/s 2024-03-07T03:17:57.618 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 44262099~8197992 2024-03-07T03:17:57.656 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 15410784~1511369 2024-03-07T03:17:57.750 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 113721520~8011945 2024-03-07T03:17:57.768 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 64786707~2780296 2024-03-07T03:17:57.805 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 46124190~3712132 2024-03-07T03:17:57.811 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 8271132~4443231 2024-03-07T03:17:57.862 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 71046266~2448638 2024-03-07T03:17:57.876 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:17:58.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:58 smithi148 bash[79819]: cluster 2024-03-07T03:17:57.118507+0000 mon.a (mon.0) 127 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2024-03-07T03:17:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:58 smithi148 bash[79819]: audit 2024-03-07T03:17:57.294030+0000 mon.a (mon.0) 128 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:58 smithi148 bash[31149]: cluster 2024-03-07T03:17:57.118507+0000 mon.a (mon.0) 127 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2024-03-07T03:17:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:58 smithi148 bash[31149]: audit 2024-03-07T03:17:57.294030+0000 mon.a (mon.0) 128 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:58 smithi195 bash[23221]: cluster 2024-03-07T03:17:57.118507+0000 mon.a (mon.0) 127 : cluster 0 osdmap e330: 8 total, 8 up, 8 in 2024-03-07T03:17:58.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:58 smithi195 bash[23221]: audit 2024-03-07T03:17:57.294030+0000 mon.a (mon.0) 128 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:17:59.157 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap3 2024-03-07T03:17:59.188 INFO:tasks.workunit.client.0.smithi148.stdout:clone image174 -> image176 2024-03-07T03:17:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:59 smithi148 bash[79819]: cluster 2024-03-07T03:17:58.119715+0000 mon.a (mon.0) 129 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2024-03-07T03:17:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:17:59 smithi148 bash[79819]: cluster 2024-03-07T03:17:58.204826+0000 mgr.y (mgr.34101) 39 : cluster 0 pgmap v71: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 26 MiB/s wr, 360 op/s 2024-03-07T03:17:59.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:59 smithi148 bash[31149]: cluster 2024-03-07T03:17:58.119715+0000 mon.a (mon.0) 129 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2024-03-07T03:17:59.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:17:59 smithi148 bash[31149]: cluster 2024-03-07T03:17:58.204826+0000 mgr.y (mgr.34101) 39 : cluster 0 pgmap v71: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 26 MiB/s wr, 360 op/s 2024-03-07T03:17:59.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:59 smithi195 bash[23221]: cluster 2024-03-07T03:17:58.119715+0000 mon.a (mon.0) 129 : cluster 0 osdmap e331: 8 total, 8 up, 8 in 2024-03-07T03:17:59.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:17:59 smithi195 bash[23221]: cluster 2024-03-07T03:17:58.204826+0000 mgr.y (mgr.34101) 39 : cluster 0 pgmap v71: 137 pgs: 137 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 9.1 MiB/s rd, 26 MiB/s wr, 360 op/s 2024-03-07T03:17:59.591 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 177382458 2024-03-07T03:17:59.632 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:18:00.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:00 smithi148 bash[79819]: cluster 2024-03-07T03:17:59.139780+0000 mon.a (mon.0) 130 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2024-03-07T03:18:00.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:00 smithi148 bash[31149]: cluster 2024-03-07T03:17:59.139780+0000 mon.a (mon.0) 130 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2024-03-07T03:18:00.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:00 smithi195 bash[23221]: cluster 2024-03-07T03:17:59.139780+0000 mon.a (mon.0) 130 : cluster 0 osdmap e332: 8 total, 8 up, 8 in 2024-03-07T03:18:00.652 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:18:01.164 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:18:01.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:01 smithi148 bash[79819]: cluster 2024-03-07T03:18:00.205287+0000 mgr.y (mgr.34101) 40 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 1.3 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 493 op/s 2024-03-07T03:18:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:01 smithi148 bash[31149]: cluster 2024-03-07T03:18:00.205287+0000 mgr.y (mgr.34101) 40 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 1.3 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 493 op/s 2024-03-07T03:18:01.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:01 smithi195 bash[23221]: cluster 2024-03-07T03:18:00.205287+0000 mgr.y (mgr.34101) 40 : cluster 0 pgmap v73: 137 pgs: 137 active+clean; 1.3 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 10 MiB/s rd, 30 MiB/s wr, 493 op/s 2024-03-07T03:18:02.010 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.Stress (11112 ms) 2024-03-07T03:18:02.010 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.Stress2 2024-03-07T03:18:02.100 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:18:02.132 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 83251846~36966 2024-03-07T03:18:02.159 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 3219328~2733637 2024-03-07T03:18:02.159 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 78328815~5557265 2024-03-07T03:18:02.210 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 81884967~2001113 2024-03-07T03:18:02.216 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 75651556~8234524 2024-03-07T03:18:02.274 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 55116718~5812304 2024-03-07T03:18:02.274 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 40362079~5417731 2024-03-07T03:18:02.326 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 3347218~4594454 2024-03-07T03:18:02.326 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap0 2024-03-07T03:18:02.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:02 smithi195 bash[59134]: ts=2024-03-07T03:18:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:03.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:03 smithi148 bash[79819]: cluster 2024-03-07T03:18:02.206038+0000 mgr.y (mgr.34101) 41 : cluster 0 pgmap v74: 137 pgs: 137 active+clean; 1.3 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 MiB/s rd, 20 MiB/s wr, 337 op/s 2024-03-07T03:18:03.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:03 smithi148 bash[31149]: cluster 2024-03-07T03:18:02.206038+0000 mgr.y (mgr.34101) 41 : cluster 0 pgmap v74: 137 pgs: 137 active+clean; 1.3 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 MiB/s rd, 20 MiB/s wr, 337 op/s 2024-03-07T03:18:03.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:03 smithi195 bash[23221]: cluster 2024-03-07T03:18:02.206038+0000 mgr.y (mgr.34101) 41 : cluster 0 pgmap v74: 137 pgs: 137 active+clean; 1.3 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 6.9 MiB/s rd, 20 MiB/s wr, 337 op/s 2024-03-07T03:18:03.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:03 smithi195 bash[59134]: ts=2024-03-07T03:18:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:04.302 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 162354107 2024-03-07T03:18:04.332 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 101244768~3189847 2024-03-07T03:18:04.359 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 95683055~1668037 2024-03-07T03:18:04.359 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 66755965~8309232 2024-03-07T03:18:04.379 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 74896179~6784503 2024-03-07T03:18:04.429 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 62382219~6129156 2024-03-07T03:18:04.450 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 33882739~4706835 2024-03-07T03:18:04.451 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 110153950~8238655 2024-03-07T03:18:04.483 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 118940717~8124727 2024-03-07T03:18:04.483 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:18:04.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:04 smithi148 bash[31149]: cluster 2024-03-07T03:18:03.276329+0000 mon.a (mon.0) 131 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2024-03-07T03:18:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:04 smithi148 bash[79819]: cluster 2024-03-07T03:18:03.276329+0000 mon.a (mon.0) 131 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2024-03-07T03:18:04.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:04 smithi195 bash[23221]: cluster 2024-03-07T03:18:03.276329+0000 mon.a (mon.0) 131 : cluster 0 osdmap e333: 8 total, 8 up, 8 in 2024-03-07T03:18:05.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:05 smithi148 bash[79819]: cluster 2024-03-07T03:18:04.206998+0000 mgr.y (mgr.34101) 42 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 953 op/s 2024-03-07T03:18:05.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:05 smithi148 bash[79819]: cluster 2024-03-07T03:18:04.291525+0000 mon.a (mon.0) 132 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2024-03-07T03:18:05.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:05 smithi148 bash[31149]: cluster 2024-03-07T03:18:04.206998+0000 mgr.y (mgr.34101) 42 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 953 op/s 2024-03-07T03:18:05.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:05 smithi148 bash[31149]: cluster 2024-03-07T03:18:04.291525+0000 mon.a (mon.0) 132 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2024-03-07T03:18:05.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:05 smithi195 bash[23221]: cluster 2024-03-07T03:18:04.206998+0000 mgr.y (mgr.34101) 42 : cluster 0 pgmap v76: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 29 MiB/s rd, 26 MiB/s wr, 953 op/s 2024-03-07T03:18:05.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:05 smithi195 bash[23221]: cluster 2024-03-07T03:18:04.291525+0000 mon.a (mon.0) 132 : cluster 0 osdmap e334: 8 total, 8 up, 8 in 2024-03-07T03:18:06.319 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap1 2024-03-07T03:18:06.349 INFO:tasks.workunit.client.0.smithi148.stdout:clone image177 -> image180 2024-03-07T03:18:06.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:06 smithi148 bash[79819]: cluster 2024-03-07T03:18:05.290956+0000 mon.a (mon.0) 133 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2024-03-07T03:18:06.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:06 smithi148 bash[31149]: cluster 2024-03-07T03:18:05.290956+0000 mon.a (mon.0) 133 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2024-03-07T03:18:06.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:06 smithi195 bash[23221]: cluster 2024-03-07T03:18:05.290956+0000 mon.a (mon.0) 133 : cluster 0 osdmap e335: 8 total, 8 up, 8 in 2024-03-07T03:18:06.623 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 85986014 2024-03-07T03:18:06.669 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 18871590~7990894 2024-03-07T03:18:06.772 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 59934979~4504024 2024-03-07T03:18:06.831 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 18288108~635282 2024-03-07T03:18:06.844 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 66005414~3833630 2024-03-07T03:18:06.889 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 77117064~4880366 2024-03-07T03:18:06.959 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 49771705~7047581 2024-03-07T03:18:06.979 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 3234858~6953885 2024-03-07T03:18:07.054 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 47365784~1130351 2024-03-07T03:18:07.059 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:18:07.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:18:07] "GET /metrics HTTP/1.1" 200 38839 "" "Prometheus/2.43.0" 2024-03-07T03:18:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:07 smithi148 bash[79819]: cluster 2024-03-07T03:18:06.207782+0000 mgr.y (mgr.34101) 43 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 23 MiB/s rd, 6.1 MiB/s wr, 629 op/s 2024-03-07T03:18:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:07 smithi148 bash[79819]: cluster 2024-03-07T03:18:06.312183+0000 mon.a (mon.0) 134 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2024-03-07T03:18:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:07 smithi148 bash[31149]: cluster 2024-03-07T03:18:06.207782+0000 mgr.y (mgr.34101) 43 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 23 MiB/s rd, 6.1 MiB/s wr, 629 op/s 2024-03-07T03:18:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:07 smithi148 bash[31149]: cluster 2024-03-07T03:18:06.312183+0000 mon.a (mon.0) 134 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2024-03-07T03:18:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:07 smithi195 bash[23221]: cluster 2024-03-07T03:18:06.207782+0000 mgr.y (mgr.34101) 43 : cluster 0 pgmap v79: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 23 MiB/s rd, 6.1 MiB/s wr, 629 op/s 2024-03-07T03:18:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:07 smithi195 bash[23221]: cluster 2024-03-07T03:18:06.312183+0000 mon.a (mon.0) 134 : cluster 0 osdmap e336: 8 total, 8 up, 8 in 2024-03-07T03:18:08.347 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap2 2024-03-07T03:18:08.375 INFO:tasks.workunit.client.0.smithi148.stdout:clone image180 -> image182 2024-03-07T03:18:08.708 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 112915055 2024-03-07T03:18:08.738 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 14806073~5547001 2024-03-07T03:18:08.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:08 smithi148 bash[79819]: cluster 2024-03-07T03:18:07.324905+0000 mon.a (mon.0) 135 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2024-03-07T03:18:08.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:08 smithi148 bash[31149]: cluster 2024-03-07T03:18:07.324905+0000 mon.a (mon.0) 135 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2024-03-07T03:18:08.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:08 smithi195 bash[23221]: cluster 2024-03-07T03:18:07.324905+0000 mon.a (mon.0) 135 : cluster 0 osdmap e337: 8 total, 8 up, 8 in 2024-03-07T03:18:08.819 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 109238823~3676232 2024-03-07T03:18:08.819 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 4299270~1984650 2024-03-07T03:18:08.906 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 69491939~5937830 2024-03-07T03:18:08.935 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 89188936~6595655 2024-03-07T03:18:08.998 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 53057563~6113672 2024-03-07T03:18:09.017 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 7262332~6781376 2024-03-07T03:18:09.082 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 45750399~7781561 2024-03-07T03:18:09.156 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:18:09.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:09 smithi148 bash[79819]: cluster 2024-03-07T03:18:08.209138+0000 mgr.y (mgr.34101) 44 : cluster 0 pgmap v82: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 18 MiB/s wr, 246 op/s 2024-03-07T03:18:09.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:09 smithi148 bash[79819]: cluster 2024-03-07T03:18:08.336429+0000 mon.a (mon.0) 136 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2024-03-07T03:18:09.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:09 smithi148 bash[31149]: cluster 2024-03-07T03:18:08.209138+0000 mgr.y (mgr.34101) 44 : cluster 0 pgmap v82: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 18 MiB/s wr, 246 op/s 2024-03-07T03:18:09.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:09 smithi148 bash[31149]: cluster 2024-03-07T03:18:08.336429+0000 mon.a (mon.0) 136 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2024-03-07T03:18:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:09 smithi195 bash[23221]: cluster 2024-03-07T03:18:08.209138+0000 mgr.y (mgr.34101) 44 : cluster 0 pgmap v82: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 3.3 MiB/s rd, 18 MiB/s wr, 246 op/s 2024-03-07T03:18:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:09 smithi195 bash[23221]: cluster 2024-03-07T03:18:08.336429+0000 mon.a (mon.0) 136 : cluster 0 osdmap e338: 8 total, 8 up, 8 in 2024-03-07T03:18:10.367 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap3 2024-03-07T03:18:10.393 INFO:tasks.workunit.client.0.smithi148.stdout:clone image182 -> image184 2024-03-07T03:18:10.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:10 smithi148 bash[79819]: cluster 2024-03-07T03:18:09.347097+0000 mon.a (mon.0) 137 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2024-03-07T03:18:10.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:10 smithi148 bash[31149]: cluster 2024-03-07T03:18:09.347097+0000 mon.a (mon.0) 137 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2024-03-07T03:18:10.800 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 183801757 2024-03-07T03:18:10.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:10 smithi195 bash[23221]: cluster 2024-03-07T03:18:09.347097+0000 mon.a (mon.0) 137 : cluster 0 osdmap e339: 8 total, 8 up, 8 in 2024-03-07T03:18:10.839 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:18:11.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:11 smithi148 bash[79819]: cluster 2024-03-07T03:18:10.209944+0000 mgr.y (mgr.34101) 45 : cluster 0 pgmap v85: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.5 MiB/s rd, 25 MiB/s wr, 318 op/s 2024-03-07T03:18:11.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:11 smithi148 bash[79819]: cluster 2024-03-07T03:18:10.356423+0000 mon.a (mon.0) 138 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2024-03-07T03:18:11.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:11 smithi148 bash[31149]: cluster 2024-03-07T03:18:10.209944+0000 mgr.y (mgr.34101) 45 : cluster 0 pgmap v85: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.5 MiB/s rd, 25 MiB/s wr, 318 op/s 2024-03-07T03:18:11.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:11 smithi148 bash[31149]: cluster 2024-03-07T03:18:10.356423+0000 mon.a (mon.0) 138 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2024-03-07T03:18:11.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:11 smithi195 bash[23221]: cluster 2024-03-07T03:18:10.209944+0000 mgr.y (mgr.34101) 45 : cluster 0 pgmap v85: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 5.5 MiB/s rd, 25 MiB/s wr, 318 op/s 2024-03-07T03:18:11.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:11 smithi195 bash[23221]: cluster 2024-03-07T03:18:10.356423+0000 mon.a (mon.0) 138 : cluster 0 osdmap e340: 8 total, 8 up, 8 in 2024-03-07T03:18:11.964 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 141519433~7828902 2024-03-07T03:18:12.033 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 3332600~3016419 2024-03-07T03:18:12.079 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 169353856~5268780 2024-03-07T03:18:12.118 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 67362244~1356382 2024-03-07T03:18:12.146 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 89167820~7048952 2024-03-07T03:18:12.240 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 151110783~5252143 2024-03-07T03:18:12.246 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 104751856~1692514 2024-03-07T03:18:12.300 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 141513566~6199727 2024-03-07T03:18:12.305 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user0 2024-03-07T03:18:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:12 smithi195 bash[23221]: audit 2024-03-07T03:18:12.298726+0000 mon.a (mon.0) 139 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:12 smithi195 bash[59134]: ts=2024-03-07T03:18:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:12.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:12 smithi148 bash[79819]: audit 2024-03-07T03:18:12.298726+0000 mon.a (mon.0) 139 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:12 smithi148 bash[31149]: audit 2024-03-07T03:18:12.298726+0000 mon.a (mon.0) 139 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:13.383 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 188129631 2024-03-07T03:18:13.417 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 170971125~1106966 2024-03-07T03:18:13.432 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 176058215~7933054 2024-03-07T03:18:13.432 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 77195395~3276186 2024-03-07T03:18:13.436 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 170660074~6678823 2024-03-07T03:18:13.484 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 35652006~2214935 2024-03-07T03:18:13.485 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 29638997~7839237 2024-03-07T03:18:13.554 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 152342064~1349955 2024-03-07T03:18:13.555 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 81195983~558895 2024-03-07T03:18:13.555 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user1 2024-03-07T03:18:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:13 smithi148 bash[79819]: cluster 2024-03-07T03:18:12.210627+0000 mgr.y (mgr.34101) 46 : cluster 0 pgmap v87: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 4.5 MiB/s rd, 20 MiB/s wr, 260 op/s 2024-03-07T03:18:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:13 smithi148 bash[79819]: cluster 2024-03-07T03:18:12.358385+0000 mon.a (mon.0) 140 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2024-03-07T03:18:13.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:13 smithi148 bash[31149]: cluster 2024-03-07T03:18:12.210627+0000 mgr.y (mgr.34101) 46 : cluster 0 pgmap v87: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 4.5 MiB/s rd, 20 MiB/s wr, 260 op/s 2024-03-07T03:18:13.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:13 smithi148 bash[31149]: cluster 2024-03-07T03:18:12.358385+0000 mon.a (mon.0) 140 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2024-03-07T03:18:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:13 smithi195 bash[23221]: cluster 2024-03-07T03:18:12.210627+0000 mgr.y (mgr.34101) 46 : cluster 0 pgmap v87: 137 pgs: 137 active+clean; 1.4 GiB data, 3.7 GiB used, 711 GiB / 715 GiB avail; 4.5 MiB/s rd, 20 MiB/s wr, 260 op/s 2024-03-07T03:18:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:13 smithi195 bash[23221]: cluster 2024-03-07T03:18:12.358385+0000 mon.a (mon.0) 140 : cluster 0 osdmap e341: 8 total, 8 up, 8 in 2024-03-07T03:18:13.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:13 smithi195 bash[59134]: ts=2024-03-07T03:18:13.512Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:14 smithi148 bash[79819]: cluster 2024-03-07T03:18:13.379242+0000 mon.a (mon.0) 141 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2024-03-07T03:18:14.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:14 smithi148 bash[31149]: cluster 2024-03-07T03:18:13.379242+0000 mon.a (mon.0) 141 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2024-03-07T03:18:14.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:14 smithi195 bash[23221]: cluster 2024-03-07T03:18:13.379242+0000 mon.a (mon.0) 141 : cluster 0 osdmap e342: 8 total, 8 up, 8 in 2024-03-07T03:18:15.399 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 198122080 2024-03-07T03:18:15.436 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 47168909~5948070 2024-03-07T03:18:15.467 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 1874174~3838199 2024-03-07T03:18:15.467 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 77424134~2880694 2024-03-07T03:18:15.470 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 112870457~7131909 2024-03-07T03:18:15.471 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 70542580~2547437 2024-03-07T03:18:15.480 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 183008830~8130952 2024-03-07T03:18:15.511 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 73612542~6280427 2024-03-07T03:18:15.599 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 168412937~3540304 2024-03-07T03:18:15.599 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user2 2024-03-07T03:18:15.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:15 smithi148 bash[79819]: cluster 2024-03-07T03:18:14.211731+0000 mgr.y (mgr.34101) 47 : cluster 0 pgmap v90: 137 pgs: 137 active+clean; 1.6 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 26 MiB/s wr, 675 op/s 2024-03-07T03:18:15.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:15 smithi148 bash[79819]: cluster 2024-03-07T03:18:14.388952+0000 mon.a (mon.0) 142 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2024-03-07T03:18:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:15 smithi148 bash[31149]: cluster 2024-03-07T03:18:14.211731+0000 mgr.y (mgr.34101) 47 : cluster 0 pgmap v90: 137 pgs: 137 active+clean; 1.6 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 26 MiB/s wr, 675 op/s 2024-03-07T03:18:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:15 smithi148 bash[31149]: cluster 2024-03-07T03:18:14.388952+0000 mon.a (mon.0) 142 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2024-03-07T03:18:15.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:15 smithi195 bash[23221]: cluster 2024-03-07T03:18:14.211731+0000 mgr.y (mgr.34101) 47 : cluster 0 pgmap v90: 137 pgs: 137 active+clean; 1.6 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 26 MiB/s wr, 675 op/s 2024-03-07T03:18:15.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:15 smithi195 bash[23221]: cluster 2024-03-07T03:18:14.388952+0000 mon.a (mon.0) 142 : cluster 0 osdmap e343: 8 total, 8 up, 8 in 2024-03-07T03:18:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:16 smithi148 bash[79819]: cluster 2024-03-07T03:18:15.390397+0000 mon.a (mon.0) 143 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2024-03-07T03:18:16.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:16 smithi148 bash[31149]: cluster 2024-03-07T03:18:15.390397+0000 mon.a (mon.0) 143 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2024-03-07T03:18:16.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:16 smithi195 bash[23221]: cluster 2024-03-07T03:18:15.390397+0000 mon.a (mon.0) 143 : cluster 0 osdmap e344: 8 total, 8 up, 8 in 2024-03-07T03:18:17.427 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 191305589~6816491 2024-03-07T03:18:17.467 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 12056562~7425581 2024-03-07T03:18:17.468 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 77102292~2473183 2024-03-07T03:18:17.473 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 110706955~3118583 2024-03-07T03:18:17.474 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 151495139~2484503 2024-03-07T03:18:17.484 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 37671070~6173115 2024-03-07T03:18:17.532 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 171384343~7388307 2024-03-07T03:18:17.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:18:17] "GET /metrics HTTP/1.1" 200 38835 "" "Prometheus/2.43.0" 2024-03-07T03:18:17.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:17 smithi148 bash[31149]: cluster 2024-03-07T03:18:16.212551+0000 mgr.y (mgr.34101) 48 : cluster 0 pgmap v93: 137 pgs: 137 active+clean; 1.6 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 32 MiB/s wr, 822 op/s 2024-03-07T03:18:17.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:17 smithi148 bash[31149]: cluster 2024-03-07T03:18:16.409711+0000 mon.a (mon.0) 144 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2024-03-07T03:18:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:17 smithi148 bash[79819]: cluster 2024-03-07T03:18:16.212551+0000 mgr.y (mgr.34101) 48 : cluster 0 pgmap v93: 137 pgs: 137 active+clean; 1.6 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 32 MiB/s wr, 822 op/s 2024-03-07T03:18:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:17 smithi148 bash[79819]: cluster 2024-03-07T03:18:16.409711+0000 mon.a (mon.0) 144 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2024-03-07T03:18:17.606 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 79723040~4410340 2024-03-07T03:18:17.609 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user3 2024-03-07T03:18:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:17 smithi195 bash[23221]: cluster 2024-03-07T03:18:16.212551+0000 mgr.y (mgr.34101) 48 : cluster 0 pgmap v93: 137 pgs: 137 active+clean; 1.6 GiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 34 MiB/s rd, 32 MiB/s wr, 822 op/s 2024-03-07T03:18:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:17 smithi195 bash[23221]: cluster 2024-03-07T03:18:16.409711+0000 mon.a (mon.0) 144 : cluster 0 osdmap e345: 8 total, 8 up, 8 in 2024-03-07T03:18:18.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:18 smithi148 bash[79819]: cluster 2024-03-07T03:18:17.418090+0000 mon.a (mon.0) 145 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2024-03-07T03:18:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:18 smithi148 bash[31149]: cluster 2024-03-07T03:18:17.418090+0000 mon.a (mon.0) 145 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2024-03-07T03:18:18.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:18 smithi195 bash[23221]: cluster 2024-03-07T03:18:17.418090+0000 mon.a (mon.0) 145 : cluster 0 osdmap e346: 8 total, 8 up, 8 in 2024-03-07T03:18:19.460 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 19812208~4953052 2024-03-07T03:18:19.478 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 21050471~2476526 2024-03-07T03:18:19.555 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 39624416~4953052 2024-03-07T03:18:19.558 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 40862679~2476526 2024-03-07T03:18:19.619 INFO:tasks.workunit.client.0.smithi148.stdout:write: 3 59436624~4953052 2024-03-07T03:18:19.622 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 60674887~2476526 2024-03-07T03:18:19.695 INFO:tasks.workunit.client.0.smithi148.stdout:write: 4 79248832~4953052 2024-03-07T03:18:19.697 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 80487095~2476526 2024-03-07T03:18:19.757 INFO:tasks.workunit.client.0.smithi148.stdout:write: 5 99061040~4953052 2024-03-07T03:18:19.760 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 100299303~2476526 2024-03-07T03:18:19.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:19 smithi148 bash[79819]: cluster 2024-03-07T03:18:18.214238+0000 mgr.y (mgr.34101) 49 : cluster 0 pgmap v96: 137 pgs: 137 active+clean; 1.7 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 37 MiB/s wr, 321 op/s 2024-03-07T03:18:19.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:19 smithi148 bash[79819]: cluster 2024-03-07T03:18:18.431472+0000 mon.a (mon.0) 146 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2024-03-07T03:18:19.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:19 smithi148 bash[31149]: cluster 2024-03-07T03:18:18.214238+0000 mgr.y (mgr.34101) 49 : cluster 0 pgmap v96: 137 pgs: 137 active+clean; 1.7 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 37 MiB/s wr, 321 op/s 2024-03-07T03:18:19.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:19 smithi148 bash[31149]: cluster 2024-03-07T03:18:18.431472+0000 mon.a (mon.0) 146 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2024-03-07T03:18:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:19 smithi195 bash[23221]: cluster 2024-03-07T03:18:18.214238+0000 mgr.y (mgr.34101) 49 : cluster 0 pgmap v96: 137 pgs: 137 active+clean; 1.7 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 13 MiB/s rd, 37 MiB/s wr, 321 op/s 2024-03-07T03:18:19.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:19 smithi195 bash[23221]: cluster 2024-03-07T03:18:18.431472+0000 mon.a (mon.0) 146 : cluster 0 osdmap e347: 8 total, 8 up, 8 in 2024-03-07T03:18:20.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:20 smithi148 bash[79819]: cluster 2024-03-07T03:18:19.430796+0000 mon.a (mon.0) 147 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2024-03-07T03:18:20.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:20 smithi148 bash[31149]: cluster 2024-03-07T03:18:19.430796+0000 mon.a (mon.0) 147 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2024-03-07T03:18:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:20 smithi195 bash[23221]: cluster 2024-03-07T03:18:19.430796+0000 mon.a (mon.0) 147 : cluster 0 osdmap e348: 8 total, 8 up, 8 in 2024-03-07T03:18:21.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:21 smithi148 bash[79819]: cluster 2024-03-07T03:18:20.214950+0000 mgr.y (mgr.34101) 50 : cluster 0 pgmap v99: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 42 MiB/s wr, 446 op/s 2024-03-07T03:18:21.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:21 smithi148 bash[31149]: cluster 2024-03-07T03:18:20.214950+0000 mgr.y (mgr.34101) 50 : cluster 0 pgmap v99: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 42 MiB/s wr, 446 op/s 2024-03-07T03:18:21.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:21 smithi195 bash[23221]: cluster 2024-03-07T03:18:20.214950+0000 mgr.y (mgr.34101) 50 : cluster 0 pgmap v99: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s rd, 42 MiB/s wr, 446 op/s 2024-03-07T03:18:22.156 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:18:22.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:22 smithi195 bash[59134]: ts=2024-03-07T03:18:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:22.722 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:18:23.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:23 smithi148 bash[79819]: cluster 2024-03-07T03:18:22.215861+0000 mgr.y (mgr.34101) 51 : cluster 0 pgmap v100: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 29 MiB/s wr, 307 op/s 2024-03-07T03:18:23.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:23 smithi148 bash[31149]: cluster 2024-03-07T03:18:22.215861+0000 mgr.y (mgr.34101) 51 : cluster 0 pgmap v100: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 29 MiB/s wr, 307 op/s 2024-03-07T03:18:23.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:23 smithi195 bash[23221]: cluster 2024-03-07T03:18:22.215861+0000 mgr.y (mgr.34101) 51 : cluster 0 pgmap v100: 137 pgs: 137 active+clean; 1.8 GiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 11 MiB/s rd, 29 MiB/s wr, 307 op/s 2024-03-07T03:18:23.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:23 smithi195 bash[59134]: ts=2024-03-07T03:18:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:25.311 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.Stress2 (23300 ms) 2024-03-07T03:18:25.311 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMigration.StressLive 2024-03-07T03:18:25.397 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 83886080 2024-03-07T03:18:25.428 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 67335831~630147 2024-03-07T03:18:25.462 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 66482312~8152117 2024-03-07T03:18:25.480 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 44253986~4800176 2024-03-07T03:18:25.529 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 12931128~4316477 2024-03-07T03:18:25.529 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 27374120~300129 2024-03-07T03:18:25.545 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 20617649~5391831 2024-03-07T03:18:25.550 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 39693021~8033169 2024-03-07T03:18:25.606 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 50234456~2268316 2024-03-07T03:18:25.611 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap0 2024-03-07T03:18:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:25 smithi148 bash[79819]: cluster 2024-03-07T03:18:24.217442+0000 mgr.y (mgr.34101) 52 : cluster 0 pgmap v101: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 131 MiB/s rd, 35 MiB/s wr, 724 op/s 2024-03-07T03:18:25.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:25 smithi148 bash[31149]: cluster 2024-03-07T03:18:24.217442+0000 mgr.y (mgr.34101) 52 : cluster 0 pgmap v101: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 131 MiB/s rd, 35 MiB/s wr, 724 op/s 2024-03-07T03:18:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:25 smithi195 bash[23221]: cluster 2024-03-07T03:18:24.217442+0000 mgr.y (mgr.34101) 52 : cluster 0 pgmap v101: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 131 MiB/s rd, 35 MiB/s wr, 724 op/s 2024-03-07T03:18:27.463 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:18:27] "GET /metrics HTTP/1.1" 200 38835 "" "Prometheus/2.43.0" 2024-03-07T03:18:27.494 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap0 2024-03-07T03:18:27.523 INFO:tasks.workunit.client.0.smithi148.stdout:clone image185 -> image188 2024-03-07T03:18:27.780 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 39864542~2375991 2024-03-07T03:18:27.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:27 smithi148 bash[79819]: cluster 2024-03-07T03:18:26.218167+0000 mgr.y (mgr.34101) 53 : cluster 0 pgmap v102: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s rd, 11 MiB/s wr, 454 op/s 2024-03-07T03:18:27.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:27 smithi148 bash[79819]: cluster 2024-03-07T03:18:26.471232+0000 mon.a (mon.0) 148 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2024-03-07T03:18:27.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:27 smithi148 bash[79819]: audit 2024-03-07T03:18:27.299253+0000 mon.a (mon.0) 149 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:27.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:27 smithi148 bash[31149]: cluster 2024-03-07T03:18:26.218167+0000 mgr.y (mgr.34101) 53 : cluster 0 pgmap v102: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s rd, 11 MiB/s wr, 454 op/s 2024-03-07T03:18:27.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:27 smithi148 bash[31149]: cluster 2024-03-07T03:18:26.471232+0000 mon.a (mon.0) 148 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2024-03-07T03:18:27.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:27 smithi148 bash[31149]: audit 2024-03-07T03:18:27.299253+0000 mon.a (mon.0) 149 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:27 smithi195 bash[23221]: cluster 2024-03-07T03:18:26.218167+0000 mgr.y (mgr.34101) 53 : cluster 0 pgmap v102: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 105 MiB/s rd, 11 MiB/s wr, 454 op/s 2024-03-07T03:18:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:27 smithi195 bash[23221]: cluster 2024-03-07T03:18:26.471232+0000 mon.a (mon.0) 148 : cluster 0 osdmap e349: 8 total, 8 up, 8 in 2024-03-07T03:18:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:27 smithi195 bash[23221]: audit 2024-03-07T03:18:27.299253+0000 mon.a (mon.0) 149 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:27.889 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 49933251~666246 2024-03-07T03:18:27.943 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 53007401~406006 2024-03-07T03:18:27.954 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 57588532~5774718 2024-03-07T03:18:27.974 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 68107157~1675452 2024-03-07T03:18:28.005 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 80607464~3278616 2024-03-07T03:18:28.019 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 62629019~60067 2024-03-07T03:18:28.025 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16600792~7637300 2024-03-07T03:18:28.045 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap1 2024-03-07T03:18:28.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:28 smithi148 bash[79819]: cluster 2024-03-07T03:18:27.473065+0000 mon.a (mon.0) 150 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2024-03-07T03:18:28.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:28 smithi148 bash[31149]: cluster 2024-03-07T03:18:27.473065+0000 mon.a (mon.0) 150 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2024-03-07T03:18:28.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:28 smithi195 bash[23221]: cluster 2024-03-07T03:18:27.473065+0000 mon.a (mon.0) 150 : cluster 0 osdmap e350: 8 total, 8 up, 8 in 2024-03-07T03:18:29.513 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 49541729~2134523 2024-03-07T03:18:29.529 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 44377692~202780 2024-03-07T03:18:29.529 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 22371827~1418377 2024-03-07T03:18:29.530 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 34617370~6275085 2024-03-07T03:18:29.531 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 67263658~3344592 2024-03-07T03:18:29.533 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 67240411~965746 2024-03-07T03:18:29.584 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 20118954~6091064 2024-03-07T03:18:29.588 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 65477566~6441657 2024-03-07T03:18:29.589 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap2 2024-03-07T03:18:29.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:29 smithi148 bash[79819]: cluster 2024-03-07T03:18:28.219914+0000 mgr.y (mgr.34101) 54 : cluster 0 pgmap v105: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 188 MiB/s rd, 14 MiB/s wr, 545 op/s 2024-03-07T03:18:29.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:29 smithi148 bash[79819]: cluster 2024-03-07T03:18:28.494490+0000 mon.a (mon.0) 151 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2024-03-07T03:18:29.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:29 smithi148 bash[31149]: cluster 2024-03-07T03:18:28.219914+0000 mgr.y (mgr.34101) 54 : cluster 0 pgmap v105: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 188 MiB/s rd, 14 MiB/s wr, 545 op/s 2024-03-07T03:18:29.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:29 smithi148 bash[31149]: cluster 2024-03-07T03:18:28.494490+0000 mon.a (mon.0) 151 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2024-03-07T03:18:29.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:29 smithi195 bash[23221]: cluster 2024-03-07T03:18:28.219914+0000 mgr.y (mgr.34101) 54 : cluster 0 pgmap v105: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 188 MiB/s rd, 14 MiB/s wr, 545 op/s 2024-03-07T03:18:29.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:29 smithi195 bash[23221]: cluster 2024-03-07T03:18:28.494490+0000 mon.a (mon.0) 151 : cluster 0 osdmap e351: 8 total, 8 up, 8 in 2024-03-07T03:18:30.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:30 smithi148 bash[79819]: cluster 2024-03-07T03:18:29.505448+0000 mon.a (mon.0) 152 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2024-03-07T03:18:30.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:30 smithi148 bash[31149]: cluster 2024-03-07T03:18:29.505448+0000 mon.a (mon.0) 152 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2024-03-07T03:18:30.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:30 smithi195 bash[23221]: cluster 2024-03-07T03:18:29.505448+0000 mon.a (mon.0) 152 : cluster 0 osdmap e352: 8 total, 8 up, 8 in 2024-03-07T03:18:31.522 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap2 2024-03-07T03:18:31.533 INFO:tasks.workunit.client.0.smithi148.stdout:clone image188 -> image190 2024-03-07T03:18:31.676 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 118484968 2024-03-07T03:18:31.696 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 70735891~329451 2024-03-07T03:18:31.716 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 67392152~2719908 2024-03-07T03:18:31.720 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 19629225~605948 2024-03-07T03:18:31.748 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 112555119~1387811 2024-03-07T03:18:31.748 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 26443132~990325 2024-03-07T03:18:31.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:31 smithi148 bash[31149]: cluster 2024-03-07T03:18:30.220673+0000 mgr.y (mgr.34101) 55 : cluster 0 pgmap v108: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 182 MiB/s rd, 12 MiB/s wr, 335 op/s 2024-03-07T03:18:31.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:31 smithi148 bash[31149]: cluster 2024-03-07T03:18:30.513238+0000 mon.a (mon.0) 153 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2024-03-07T03:18:31.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:31 smithi148 bash[31149]: audit 2024-03-07T03:18:31.094643+0000 mon.a (mon.0) 154 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:31 smithi148 bash[79819]: cluster 2024-03-07T03:18:30.220673+0000 mgr.y (mgr.34101) 55 : cluster 0 pgmap v108: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 182 MiB/s rd, 12 MiB/s wr, 335 op/s 2024-03-07T03:18:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:31 smithi148 bash[79819]: cluster 2024-03-07T03:18:30.513238+0000 mon.a (mon.0) 153 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2024-03-07T03:18:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:31 smithi148 bash[79819]: audit 2024-03-07T03:18:31.094643+0000 mon.a (mon.0) 154 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:31.790 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 34448761~7796923 2024-03-07T03:18:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:31 smithi195 bash[23221]: cluster 2024-03-07T03:18:30.220673+0000 mgr.y (mgr.34101) 55 : cluster 0 pgmap v108: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 182 MiB/s rd, 12 MiB/s wr, 335 op/s 2024-03-07T03:18:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:31 smithi195 bash[23221]: cluster 2024-03-07T03:18:30.513238+0000 mon.a (mon.0) 153 : cluster 0 osdmap e353: 8 total, 8 up, 8 in 2024-03-07T03:18:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:31 smithi195 bash[23221]: audit 2024-03-07T03:18:31.094643+0000 mon.a (mon.0) 154 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:31.851 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 81525940~3345911 2024-03-07T03:18:31.891 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 70833475~5241974 2024-03-07T03:18:31.908 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: snap3 2024-03-07T03:18:32.104 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade resume' 2024-03-07T03:18:32.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:32 smithi195 bash[23221]: cluster 2024-03-07T03:18:31.522454+0000 mon.a (mon.0) 155 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2024-03-07T03:18:32.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:32 smithi195 bash[59134]: ts=2024-03-07T03:18:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:32.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:32 smithi148 bash[79819]: cluster 2024-03-07T03:18:31.522454+0000 mon.a (mon.0) 155 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2024-03-07T03:18:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:32 smithi148 bash[31149]: cluster 2024-03-07T03:18:31.522454+0000 mon.a (mon.0) 155 : cluster 0 osdmap e354: 8 total, 8 up, 8 in 2024-03-07T03:18:33.556 INFO:tasks.workunit.client.0.smithi148.stdout:snap_protect: snap3 2024-03-07T03:18:33.581 INFO:tasks.workunit.client.0.smithi148.stdout:clone image190 -> image192 2024-03-07T03:18:33.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:33 smithi148 bash[31149]: cluster 2024-03-07T03:18:32.221166+0000 mgr.y (mgr.34101) 56 : cluster 0 pgmap v111: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 28 op/s 2024-03-07T03:18:33.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:33 smithi148 bash[31149]: cluster 2024-03-07T03:18:32.534919+0000 mon.a (mon.0) 156 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2024-03-07T03:18:33.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:33 smithi148 bash[79819]: cluster 2024-03-07T03:18:32.221166+0000 mgr.y (mgr.34101) 56 : cluster 0 pgmap v111: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 28 op/s 2024-03-07T03:18:33.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:33 smithi148 bash[79819]: cluster 2024-03-07T03:18:32.534919+0000 mon.a (mon.0) 156 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2024-03-07T03:18:33.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:33 smithi195 bash[23221]: cluster 2024-03-07T03:18:32.221166+0000 mgr.y (mgr.34101) 56 : cluster 0 pgmap v111: 137 pgs: 137 active+clean; 1.9 GiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 13 MiB/s rd, 2.4 MiB/s wr, 28 op/s 2024-03-07T03:18:33.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:33 smithi195 bash[23221]: cluster 2024-03-07T03:18:32.534919+0000 mon.a (mon.0) 156 : cluster 0 osdmap e355: 8 total, 8 up, 8 in 2024-03-07T03:18:33.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:33 smithi195 bash[59134]: ts=2024-03-07T03:18:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:33.886 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 184071552 2024-03-07T03:18:33.925 INFO:tasks.workunit.client.0.smithi148.stdout:migration_prepare 2024-03-07T03:18:34.762 INFO:tasks.workunit.client.0.smithi148.stdout:migration_execute 2024-03-07T03:18:34.762 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 44125242~1099711 2024-03-07T03:18:34.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:34 smithi148 bash[79819]: cluster 2024-03-07T03:18:33.548716+0000 mon.a (mon.0) 157 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2024-03-07T03:18:34.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:34 smithi148 bash[31149]: cluster 2024-03-07T03:18:33.548716+0000 mon.a (mon.0) 157 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2024-03-07T03:18:34.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:34 smithi195 bash[23221]: cluster 2024-03-07T03:18:33.548716+0000 mon.a (mon.0) 157 : cluster 0 osdmap e356: 8 total, 8 up, 8 in 2024-03-07T03:18:34.826 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 45919794~3821834 2024-03-07T03:18:34.864 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 164252611~5924117 2024-03-07T03:18:34.901 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 366798~5407196 2024-03-07T03:18:34.938 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 105403465~8281601 2024-03-07T03:18:35.010 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 130111337~2436222 2024-03-07T03:18:35.017 INFO:tasks.workunit.client.0.smithi148.stdout:write: a 67350241~291380 2024-03-07T03:18:35.054 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 89805620~3657358 2024-03-07T03:18:35.099 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user0 2024-03-07T03:18:35.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:35 smithi195 bash[23221]: cluster 2024-03-07T03:18:34.221569+0000 mgr.y (mgr.34101) 57 : cluster 0 pgmap v114: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s rd, 12 MiB/s wr, 318 op/s 2024-03-07T03:18:36.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:35 smithi148 bash[79819]: cluster 2024-03-07T03:18:34.221569+0000 mgr.y (mgr.34101) 57 : cluster 0 pgmap v114: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s rd, 12 MiB/s wr, 318 op/s 2024-03-07T03:18:36.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:35 smithi148 bash[31149]: cluster 2024-03-07T03:18:34.221569+0000 mgr.y (mgr.34101) 57 : cluster 0 pgmap v114: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s rd, 12 MiB/s wr, 318 op/s 2024-03-07T03:18:36.134 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:18:36.568 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 282700879 2024-03-07T03:18:36.582 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 20808200~3006935 2024-03-07T03:18:36.597 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 104881619~6389317 2024-03-07T03:18:36.601 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 194271428~4292528 2024-03-07T03:18:36.609 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 200872495~7564234 2024-03-07T03:18:36.611 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 4956189~8281876 2024-03-07T03:18:36.640 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 164236591~2947698 2024-03-07T03:18:36.646 INFO:tasks.workunit.client.0.smithi148.stdout:write: b 97127433~234622 2024-03-07T03:18:36.647 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 247326088~8133816 2024-03-07T03:18:36.658 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user1 2024-03-07T03:18:36.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:36 smithi195 bash[23221]: cluster 2024-03-07T03:18:35.568123+0000 mon.a (mon.0) 158 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2024-03-07T03:18:36.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:36 smithi195 bash[23221]: cluster 2024-03-07T03:18:36.222156+0000 mgr.y (mgr.34101) 58 : cluster 0 pgmap v116: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 9.9 MiB/s wr, 271 op/s 2024-03-07T03:18:37.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:36 smithi148 bash[31149]: cluster 2024-03-07T03:18:35.568123+0000 mon.a (mon.0) 158 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2024-03-07T03:18:37.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:36 smithi148 bash[31149]: cluster 2024-03-07T03:18:36.222156+0000 mgr.y (mgr.34101) 58 : cluster 0 pgmap v116: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 9.9 MiB/s wr, 271 op/s 2024-03-07T03:18:37.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:36 smithi148 bash[79819]: cluster 2024-03-07T03:18:35.568123+0000 mon.a (mon.0) 158 : cluster 0 osdmap e357: 8 total, 8 up, 8 in 2024-03-07T03:18:37.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:36 smithi148 bash[79819]: cluster 2024-03-07T03:18:36.222156+0000 mgr.y (mgr.34101) 58 : cluster 0 pgmap v116: 137 pgs: 137 active+clean; 2.0 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.8 MiB/s rd, 9.9 MiB/s wr, 271 op/s 2024-03-07T03:18:37.454 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:18:37] "GET /metrics HTTP/1.1" 200 38839 "" "Prometheus/2.43.0" 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[79819]: cluster 2024-03-07T03:18:36.572219+0000 mon.a (mon.0) 159 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[79819]: audit 2024-03-07T03:18:36.983610+0000 mon.a (mon.0) 160 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[79819]: audit 2024-03-07T03:18:36.988558+0000 mon.a (mon.0) 161 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[79819]: audit 2024-03-07T03:18:37.243359+0000 mon.a (mon.0) 162 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[79819]: audit 2024-03-07T03:18:37.253446+0000 mon.a (mon.0) 163 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[31149]: cluster 2024-03-07T03:18:36.572219+0000 mon.a (mon.0) 159 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[31149]: audit 2024-03-07T03:18:36.983610+0000 mon.a (mon.0) 160 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[31149]: audit 2024-03-07T03:18:36.988558+0000 mon.a (mon.0) 161 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[31149]: audit 2024-03-07T03:18:37.243359+0000 mon.a (mon.0) 162 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:37 smithi148 bash[31149]: audit 2024-03-07T03:18:37.253446+0000 mon.a (mon.0) 163 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:37 smithi195 bash[23221]: cluster 2024-03-07T03:18:36.572219+0000 mon.a (mon.0) 159 : cluster 0 osdmap e358: 8 total, 8 up, 8 in 2024-03-07T03:18:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:37 smithi195 bash[23221]: audit 2024-03-07T03:18:36.983610+0000 mon.a (mon.0) 160 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:37 smithi195 bash[23221]: audit 2024-03-07T03:18:36.988558+0000 mon.a (mon.0) 161 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:37 smithi195 bash[23221]: audit 2024-03-07T03:18:37.243359+0000 mon.a (mon.0) 162 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:37 smithi195 bash[23221]: audit 2024-03-07T03:18:37.253446+0000 mon.a (mon.0) 163 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:38.604 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 3874289~5669325 2024-03-07T03:18:38.640 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 101921179~3131456 2024-03-07T03:18:38.640 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 245509788~5473741 2024-03-07T03:18:38.662 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 50012794~3694105 2024-03-07T03:18:38.663 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 14358060~3384765 2024-03-07T03:18:38.672 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 105667185~1679940 2024-03-07T03:18:38.677 INFO:tasks.workunit.client.0.smithi148.stdout:write: c 34756549~3827160 2024-03-07T03:18:38.682 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 7858144~8010824 2024-03-07T03:18:38.725 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user2 2024-03-07T03:18:38.901 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[79819]: cluster 2024-03-07T03:18:37.579269+0000 mon.a (mon.0) 164 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2024-03-07T03:18:38.901 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[79819]: audit 2024-03-07T03:18:37.832098+0000 mon.a (mon.0) 165 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:38.901 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[79819]: audit 2024-03-07T03:18:37.833715+0000 mon.a (mon.0) 166 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:38.901 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[79819]: audit 2024-03-07T03:18:37.843249+0000 mon.a (mon.0) 167 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:38.902 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[79819]: cluster 2024-03-07T03:18:38.223049+0000 mgr.y (mgr.34101) 59 : cluster 0 pgmap v119: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 21 MiB/s wr, 635 op/s 2024-03-07T03:18:38.902 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[31149]: cluster 2024-03-07T03:18:37.579269+0000 mon.a (mon.0) 164 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2024-03-07T03:18:38.902 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[31149]: audit 2024-03-07T03:18:37.832098+0000 mon.a (mon.0) 165 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:38.902 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[31149]: audit 2024-03-07T03:18:37.833715+0000 mon.a (mon.0) 166 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:38.902 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[31149]: audit 2024-03-07T03:18:37.843249+0000 mon.a (mon.0) 167 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:38.902 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:38 smithi148 bash[31149]: cluster 2024-03-07T03:18:38.223049+0000 mgr.y (mgr.34101) 59 : cluster 0 pgmap v119: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 21 MiB/s wr, 635 op/s 2024-03-07T03:18:39.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:38 smithi195 bash[23221]: cluster 2024-03-07T03:18:37.579269+0000 mon.a (mon.0) 164 : cluster 0 osdmap e359: 8 total, 8 up, 8 in 2024-03-07T03:18:39.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:38 smithi195 bash[23221]: audit 2024-03-07T03:18:37.832098+0000 mon.a (mon.0) 165 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:39.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:38 smithi195 bash[23221]: audit 2024-03-07T03:18:37.833715+0000 mon.a (mon.0) 166 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:39.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:38 smithi195 bash[23221]: audit 2024-03-07T03:18:37.843249+0000 mon.a (mon.0) 167 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:39.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:38 smithi195 bash[23221]: cluster 2024-03-07T03:18:38.223049+0000 mgr.y (mgr.34101) 59 : cluster 0 pgmap v119: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 14 MiB/s rd, 21 MiB/s wr, 635 op/s 2024-03-07T03:18:39.317 INFO:teuthology.orchestra.run.smithi148.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: cluster 2024-03-07T03:18:38.583790+0000 mon.a (mon.0) 168 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2024-03-07T03:18:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.323052+0000 mgr.y (mgr.34101) 60 : audit 0 from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:18:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: cephadm 2024-03-07T03:18:39.323477+0000 mgr.y (mgr.34101) 61 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.330093+0000 mon.a (mon.0) 169 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.351151+0000 mon.a (mon.0) 170 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.352887+0000 mon.a (mon.0) 171 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.353630+0000 mon.a (mon.0) 172 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.358411+0000 mon.a (mon.0) 173 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.419178+0000 mon.a (mon.0) 174 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.421414+0000 mon.a (mon.0) 175 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.422778+0000 mon.a (mon.0) 176 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: audit 2024-03-07T03:18:39.423670+0000 mon.a (mon.0) 177 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[79819]: cephadm 2024-03-07T03:18:39.424224+0000 mgr.y (mgr.34101) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: cluster 2024-03-07T03:18:38.583790+0000 mon.a (mon.0) 168 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.323052+0000 mgr.y (mgr.34101) 60 : audit 0 from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: cephadm 2024-03-07T03:18:39.323477+0000 mgr.y (mgr.34101) 61 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:40.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.330093+0000 mon.a (mon.0) 169 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.351151+0000 mon.a (mon.0) 170 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.352887+0000 mon.a (mon.0) 171 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.353630+0000 mon.a (mon.0) 172 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.358411+0000 mon.a (mon.0) 173 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.419178+0000 mon.a (mon.0) 174 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.421414+0000 mon.a (mon.0) 175 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.422778+0000 mon.a (mon.0) 176 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: audit 2024-03-07T03:18:39.423670+0000 mon.a (mon.0) 177 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-07T03:18:40.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:39 smithi148 bash[31149]: cephadm 2024-03-07T03:18:39.424224+0000 mgr.y (mgr.34101) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-03-07T03:18:40.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: cluster 2024-03-07T03:18:38.583790+0000 mon.a (mon.0) 168 : cluster 0 osdmap e360: 8 total, 8 up, 8 in 2024-03-07T03:18:40.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.323052+0000 mgr.y (mgr.34101) 60 : audit 0 from='client.34158 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:18:40.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: cephadm 2024-03-07T03:18:39.323477+0000 mgr.y (mgr.34101) 61 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:18:40.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.330093+0000 mon.a (mon.0) 169 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:40.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.351151+0000 mon.a (mon.0) 170 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.352887+0000 mon.a (mon.0) 171 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.353630+0000 mon.a (mon.0) 172 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.358411+0000 mon.a (mon.0) 173 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.419178+0000 mon.a (mon.0) 174 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.421414+0000 mon.a (mon.0) 175 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.422778+0000 mon.a (mon.0) 176 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: audit 2024-03-07T03:18:39.423670+0000 mon.a (mon.0) 177 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-07T03:18:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:39 smithi195 bash[23221]: cephadm 2024-03-07T03:18:39.424224+0000 mgr.y (mgr.34101) 62 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-03-07T03:18:40.491 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for majority of mons to upgrade' 2024-03-07T03:18:40.619 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 442082125 2024-03-07T03:18:40.637 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 52576682~4615294 2024-03-07T03:18:40.654 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 292738891~3016166 2024-03-07T03:18:40.654 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 116177949~8012812 2024-03-07T03:18:40.668 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 79977021~2085309 2024-03-07T03:18:40.668 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 218362018~8206978 2024-03-07T03:18:40.681 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 185063792~1710658 2024-03-07T03:18:40.682 INFO:tasks.workunit.client.0.smithi148.stdout:write: d 230511989~3856979 2024-03-07T03:18:40.683 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 76611398~1066966 2024-03-07T03:18:40.683 INFO:tasks.workunit.client.0.smithi148.stdout:snap_create: user3 2024-03-07T03:18:40.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:40 smithi148 bash[79819]: cluster 2024-03-07T03:18:39.601486+0000 mon.a (mon.0) 178 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2024-03-07T03:18:40.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:40 smithi148 bash[79819]: cluster 2024-03-07T03:18:40.223761+0000 mgr.y (mgr.34101) 63 : cluster 0 pgmap v122: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 27 MiB/s wr, 832 op/s 2024-03-07T03:18:40.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:40 smithi148 bash[31149]: cluster 2024-03-07T03:18:39.601486+0000 mon.a (mon.0) 178 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2024-03-07T03:18:40.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:40 smithi148 bash[31149]: cluster 2024-03-07T03:18:40.223761+0000 mgr.y (mgr.34101) 63 : cluster 0 pgmap v122: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 27 MiB/s wr, 832 op/s 2024-03-07T03:18:41.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:40 smithi195 bash[23221]: cluster 2024-03-07T03:18:39.601486+0000 mon.a (mon.0) 178 : cluster 0 osdmap e361: 8 total, 8 up, 8 in 2024-03-07T03:18:41.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:40 smithi195 bash[23221]: cluster 2024-03-07T03:18:40.223761+0000 mgr.y (mgr.34101) 63 : cluster 0 pgmap v122: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 17 MiB/s rd, 27 MiB/s wr, 832 op/s 2024-03-07T03:18:42.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:41 smithi148 bash[31149]: cluster 2024-03-07T03:18:40.619781+0000 mon.a (mon.0) 179 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2024-03-07T03:18:42.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:41 smithi148 bash[79819]: cluster 2024-03-07T03:18:40.619781+0000 mon.a (mon.0) 179 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2024-03-07T03:18:42.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:41 smithi195 bash[23221]: cluster 2024-03-07T03:18:40.619781+0000 mon.a (mon.0) 179 : cluster 0 osdmap e362: 8 total, 8 up, 8 in 2024-03-07T03:18:42.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[59134]: ts=2024-03-07T03:18:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:42.665 INFO:tasks.workunit.client.0.smithi148.stdout:write: 1 44208212~11052053 2024-03-07T03:18:42.698 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 46971225~5526026 2024-03-07T03:18:42.748 INFO:tasks.workunit.client.0.smithi148.stdout:write: 2 88416425~11052053 2024-03-07T03:18:42.766 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 91179438~5526026 2024-03-07T03:18:42.820 INFO:tasks.workunit.client.0.smithi148.stdout:write: 3 132624637~11052053 2024-03-07T03:18:42.851 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 135387650~5526026 2024-03-07T03:18:42.900 INFO:tasks.workunit.client.0.smithi148.stdout:write: 4 176832850~11052053 2024-03-07T03:18:42.928 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 179595863~5526026 2024-03-07T03:18:42.966 INFO:tasks.workunit.client.0.smithi148.stdout:write: 5 221041062~11052053 2024-03-07T03:18:42.989 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 223804075~5526026 2024-03-07T03:18:43.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: cluster 2024-03-07T03:18:41.624418+0000 mon.a (mon.0) 180 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: cephadm 2024-03-07T03:18:41.893132+0000 mgr.y (mgr.34101) 64 : cephadm 1 Upgrade: Updating mon.c 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: audit 2024-03-07T03:18:41.898560+0000 mon.a (mon.0) 181 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: audit 2024-03-07T03:18:41.901386+0000 mon.a (mon.0) 182 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: audit 2024-03-07T03:18:41.903499+0000 mon.a (mon.0) 183 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: audit 2024-03-07T03:18:41.905381+0000 mon.a (mon.0) 184 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: cephadm 2024-03-07T03:18:41.907268+0000 mgr.y (mgr.34101) 65 : cephadm 1 Deploying daemon mon.c on smithi148 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: cluster 2024-03-07T03:18:42.224281+0000 mgr.y (mgr.34101) 66 : cluster 0 pgmap v125: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s rd, 2.8 MiB/s wr, 89 op/s 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[31149]: audit 2024-03-07T03:18:42.299403+0000 mon.a (mon.0) 185 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: cluster 2024-03-07T03:18:41.624418+0000 mon.a (mon.0) 180 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2024-03-07T03:18:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: cephadm 2024-03-07T03:18:41.893132+0000 mgr.y (mgr.34101) 64 : cephadm 1 Upgrade: Updating mon.c 2024-03-07T03:18:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: audit 2024-03-07T03:18:41.898560+0000 mon.a (mon.0) 181 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: audit 2024-03-07T03:18:41.901386+0000 mon.a (mon.0) 182 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:18:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: audit 2024-03-07T03:18:41.903499+0000 mon.a (mon.0) 183 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:18:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: audit 2024-03-07T03:18:41.905381+0000 mon.a (mon.0) 184 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: cephadm 2024-03-07T03:18:41.907268+0000 mgr.y (mgr.34101) 65 : cephadm 1 Deploying daemon mon.c on smithi148 2024-03-07T03:18:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: cluster 2024-03-07T03:18:42.224281+0000 mgr.y (mgr.34101) 66 : cluster 0 pgmap v125: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s rd, 2.8 MiB/s wr, 89 op/s 2024-03-07T03:18:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:42 smithi148 bash[79819]: audit 2024-03-07T03:18:42.299403+0000 mon.a (mon.0) 185 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: cluster 2024-03-07T03:18:41.624418+0000 mon.a (mon.0) 180 : cluster 0 osdmap e363: 8 total, 8 up, 8 in 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: cephadm 2024-03-07T03:18:41.893132+0000 mgr.y (mgr.34101) 64 : cephadm 1 Upgrade: Updating mon.c 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: audit 2024-03-07T03:18:41.898560+0000 mon.a (mon.0) 181 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: audit 2024-03-07T03:18:41.901386+0000 mon.a (mon.0) 182 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: audit 2024-03-07T03:18:41.903499+0000 mon.a (mon.0) 183 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: audit 2024-03-07T03:18:41.905381+0000 mon.a (mon.0) 184 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: cephadm 2024-03-07T03:18:41.907268+0000 mgr.y (mgr.34101) 65 : cephadm 1 Deploying daemon mon.c on smithi148 2024-03-07T03:18:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: cluster 2024-03-07T03:18:42.224281+0000 mgr.y (mgr.34101) 66 : cluster 0 pgmap v125: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 59 KiB/s rd, 2.8 MiB/s wr, 89 op/s 2024-03-07T03:18:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:42 smithi195 bash[23221]: audit 2024-03-07T03:18:42.299403+0000 mon.a (mon.0) 185 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:43.170 INFO:tasks.workunit.client.0.smithi148.stdout:migration_commit 2024-03-07T03:18:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:43 smithi148 bash[79819]: cluster 2024-03-07T03:18:42.637855+0000 mon.a (mon.0) 186 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:18:43.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:43 smithi148 bash[31149]: cluster 2024-03-07T03:18:42.637855+0000 mon.a (mon.0) 186 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:18:43.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:43 smithi195 bash[23221]: cluster 2024-03-07T03:18:42.637855+0000 mon.a (mon.0) 186 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:18:43.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:43 smithi195 bash[59134]: ts=2024-03-07T03:18:43.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:44.464 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.464 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.465 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.466 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.466 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.467 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.467 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.468 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.468 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:44 smithi148 systemd[1]: Stopping Ceph mon.c for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:18:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:44 smithi148 bash[31149]: debug 2024-03-07T03:18:44.598+0000 7f83e6e5d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:18:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:44 smithi148 bash[31149]: debug 2024-03-07T03:18:44.598+0000 7f83e6e5d700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-03-07T03:18:45.386 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMigration.StressLive (20075 ms) 2024-03-07T03:18:45.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:45 smithi148 bash[82103]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-c 2024-03-07T03:18:45.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:45 smithi148 bash[82165]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-c 2024-03-07T03:18:45.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.c.service: Deactivated successfully. 2024-03-07T03:18:45.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: Stopped Ceph mon.c for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:18:45.951 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:18:45.974 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.975 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.975 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.975 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.976 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.976 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.976 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.976 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:45.977 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: Started Ceph mon.c for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:18:45.977 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:18:45 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:18:47.247 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:18:47] "GET /metrics HTTP/1.1" 200 38833 "" "Prometheus/2.43.0" 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.049+0000 7f3fb72bbc80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.049+0000 7f3fb72bbc80 0 ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev), process ceph-mon, pid 7 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.049+0000 7f3fb72bbc80 0 pidfile_write: ignore empty --pid-file 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 0 load: jerasure load: lrc 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Git sha 0 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Compile date 2024-03-06 19:59:24 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: DB SUMMARY 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: DB Session ID: 9D6EDXIQH92NKSE5J0V2 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T03:18:47.248 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 2628 Bytes 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000042.sst 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000040.log size: 898946 ; 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.env: 0x55f65aed4b20 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.info_log: 0x55f65c18d920 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T03:18:47.249 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.statistics: (nil) 2024-03-07T03:18:47.250 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.use_fsync: 0 2024-03-07T03:18:47.250 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T03:18:47.250 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T03:18:47.250 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T03:18:47.250 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T03:18:47.250 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.db_log_dir: 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.wal_dir: 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T03:18:47.251 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.write_buffer_manager: 0x55f65c160dc0 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.unordered_write: 0 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T03:18:47.252 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.row_cache: None 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.wal_filter: None 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.wal_compression: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T03:18:47.253 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_open_files: -1 2024-03-07T03:18:47.254 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Compression algorithms supported: 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kZSTD supported: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T03:18:47.255 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T03:18:47.256 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.merge_operator: 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_filter: None 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f65c18da20) 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: cache_index_and_filter_blocks: 1 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: pin_top_level_index_and_filter: 1 2024-03-07T03:18:47.257 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: index_type: 0 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: data_block_index_type: 0 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: index_shortening: 1 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: checksum: 4 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: no_block_cache: 0 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_cache: 0x55f65c191090 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_cache_name: BinnedLRUCache 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_cache_options: 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: capacity : 536870912 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: num_shard_bits : 4 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: strict_capacity_limit : 0 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: high_pri_pool_ratio: 0.000 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_cache_compressed: (nil) 2024-03-07T03:18:47.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: persistent_cache: (nil) 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_size: 4096 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_size_deviation: 10 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_restart_interval: 16 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: index_block_restart_interval: 1 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: metadata_block_size: 4096 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: partition_filters: 0 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: use_delta_encoding: 1 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: filter_policy: bloomfilter 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: whole_key_filtering: 1 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: verify_compression: 0 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: read_amp_bytes_per_bit: 0 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: format_version: 5 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: enable_index_compression: 1 2024-03-07T03:18:47.259 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: block_align: 0 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: max_auto_readahead_size: 262144 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: prepopulate_block_cache: 0 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: initial_auto_readahead_size: 8192 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: num_file_reads_for_auto_readahead: 2 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression: NoCompression 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.num_levels: 7 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T03:18:47.260 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:18:47.261 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T03:18:47.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T03:18:47.263 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T03:18:47.264 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.ttl: 2592000 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T03:18:47.265 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 44, last_sequence is 24318, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 40 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 84778763-f66c-4eef-a714-30fd543d8382 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781527060078, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.053+0000 7f3fb72bbc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.057+0000 7f3fb72bbc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781527063740, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 587321, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24324, "largest_seqno": 24645, "table_properties": {"data_size": 584839, "index_size": 1084, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3412, "raw_average_key_size": 24, "raw_value_size": 581418, "raw_average_value_size": 4094, "num_data_blocks": 46, "num_entries": 142, "num_filter_entries": 142, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709781527, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "84778763-f66c-4eef-a714-30fd543d8382", "db_session_id": "9D6EDXIQH92NKSE5J0V2", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.057+0000 7f3fb72bbc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781527063843, "job": 1, "event": "recovery_finished"} 2024-03-07T03:18:47.266 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.057+0000 7f3fb72bbc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.057+0000 7f3fb72bbc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55f65c288000 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 4 rocksdb: DB pointer 0x55f65c272000 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fa5a24700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fa5a24700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: ** DB Stats ** 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T03:18:47.267 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: ** Compaction Stats [default] ** 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: L0 1/0 573.56 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 297.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: L6 1/0 11.70 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Sum 2/0 12.26 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 297.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 297.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: ** Compaction Stats [default] ** 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 297.5 0.00 0.00 1 0.002 0 0 0.0 0.0 2024-03-07T03:18:47.268 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Flush(GB): cumulative 0.001, interval 0.001 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Cumulative compaction: 0.00 GB write, 85.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Interval compaction: 0.00 GB write, 85.88 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:18:47.269 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Block cache BinnedLRUCache@0x55f65c191090#7 capacity: 512.00 MB usage: 1.64 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,1.20 KB,0.000229478%) Misc(1,0.00 KB,0%) 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0] at bind addrs [v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 1 mon.c@-1(???) e3 preinit fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 0 mon.c@-1(???).mds e10 new map 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 0 mon.c@-1(???).mds e10 print_map 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: e10 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: legacy client fscid: 1 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Filesystem 'foo' (1) 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: fs_name foo 2024-03-07T03:18:47.270 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: epoch 10 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: created 2024-03-07T03:11:17.721445+0000 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: modified 2024-03-07T03:11:28.139230+0000 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: tableserver 0 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: root 0 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: session_timeout 60 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: session_autoclose 300 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: max_file_size 1099511627776 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: max_xattr_size 65536 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: required_client_features {} 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: last_failure 0 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: last_failure_osd_epoch 70 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: max_mds 1 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: in 0 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: up {0=24385} 2024-03-07T03:18:47.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: failed 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: damaged 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: stopped 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: data_pools [4] 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: metadata_pool 3 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: inline_data disabled 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: balancer 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: bal_rank_mask -1 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: standby_count_wanted 1 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: [mds.foo.smithi195.gtlmjn{0:24385} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: Standby daemons: 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: 2024-03-07T03:18:47.272 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: [mds.foo.smithi148.qizikc{-1:14580} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.148:6834/4101454543,v1:172.21.15.148:6835/4101454543] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:18:47.273 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 0 mon.c@-1(???).osd e364 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-07T03:18:47.273 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 0 mon.c@-1(???).osd e364 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:18:47.273 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 0 mon.c@-1(???).osd e364 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:18:47.273 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.061+0000 7f3fb72bbc80 0 mon.c@-1(???).osd e364 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:18:47.273 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:47 smithi148 bash[82222]: debug 2024-03-07T03:18:47.065+0000 7f3fb72bbc80 1 mon.c@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-03-07T03:18:48.613 INFO:teuthology.orchestra.run.smithi148.stdout:wait for majority of mons to upgrade 2024-03-07T03:18:49.224 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-03-07T03:18:52.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:52 smithi195 bash[59134]: ts=2024-03-07T03:18:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 1/3 mons down, quorum a,b; 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-03-07T03:18:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T03:18:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : mon.c (rank 2) addr [v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0] is down (out of quorum) 2024-03-07T03:18:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:18:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:18:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:52 smithi148 bash[79819]: debug 2024-03-07T03:18:52.329+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:18:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.283296+0000 mon.b (mon.1) 205 : cluster 1 mon.b calling monitor election 2024-03-07T03:18:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.284418+0000 mon.a (mon.0) 213 : cluster 1 mon.a calling monitor election 2024-03-07T03:18:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.323080+0000 mon.a (mon.0) 214 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:18:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.330254+0000 mon.a (mon.0) 215 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:18:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.331328+0000 mon.a (mon.0) 216 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:18:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.331394+0000 mon.a (mon.0) 217 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:18:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.332111+0000 mon.a (mon.0) 218 : cluster 0 mgrmap e38: y(active, since 101s), standbys: x 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.332518+0000 mon.a (mon.0) 219 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: debug 2024-03-07T03:18:53.353+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.283296+0000 mon.b (mon.1) 205 : cluster 1 mon.b calling monitor election 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.284418+0000 mon.a (mon.0) 213 : cluster 1 mon.a calling monitor election 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.323080+0000 mon.a (mon.0) 214 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.330254+0000 mon.a (mon.0) 215 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.331328+0000 mon.a (mon.0) 216 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.331394+0000 mon.a (mon.0) 217 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.332111+0000 mon.a (mon.0) 218 : cluster 0 mgrmap e38: y(active, since 101s), standbys: x 2024-03-07T03:18:53.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:53 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.332518+0000 mon.a (mon.0) 219 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:18:53.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[59134]: ts=2024-03-07T03:18:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:18:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.283296+0000 mon.b (mon.1) 205 : cluster 1 mon.b calling monitor election 2024-03-07T03:18:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.284418+0000 mon.a (mon.0) 213 : cluster 1 mon.a calling monitor election 2024-03-07T03:18:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.323080+0000 mon.a (mon.0) 214 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:18:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.330254+0000 mon.a (mon.0) 215 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:18:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.331328+0000 mon.a (mon.0) 216 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:18:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.331394+0000 mon.a (mon.0) 217 : cluster 0 osdmap e364: 8 total, 8 up, 8 in 2024-03-07T03:18:53.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.332111+0000 mon.a (mon.0) 218 : cluster 0 mgrmap e38: y(active, since 101s), standbys: x 2024-03-07T03:18:53.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:53 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.332518+0000 mon.a (mon.0) 219 : cluster 1 Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-07T03:18:53.958 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:18:54.387 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 57 tests from TestMigration (292815 ms total) 2024-03-07T03:18:54.387 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:18:54.387 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 6 tests from TestGroup 2024-03-07T03:18:54.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:47.278315+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359145+0000 mon.a (mon.0) 220 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359189+0000 mon.a (mon.0) 221 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359209+0000 mon.a (mon.0) 222 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359230+0000 mon.a (mon.0) 223 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359280+0000 mon.a (mon.0) 224 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359336+0000 mon.a (mon.0) 225 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359380+0000 mon.a (mon.0) 226 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359415+0000 mon.a (mon.0) 227 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359525+0000 mon.a (mon.0) 228 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359574+0000 mon.a (mon.0) 229 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359609+0000 mon.a (mon.0) 230 : cluster 4 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359638+0000 mon.a (mon.0) 231 : cluster 4 application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.359667+0000 mon.a (mon.0) 232 : cluster 4 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:18:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[79819]: cluster 2024-03-07T03:18:53.370950+0000 mon.a (mon.0) 233 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:47.278315+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359145+0000 mon.a (mon.0) 220 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359189+0000 mon.a (mon.0) 221 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359209+0000 mon.a (mon.0) 222 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359230+0000 mon.a (mon.0) 223 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359280+0000 mon.a (mon.0) 224 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359336+0000 mon.a (mon.0) 225 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359380+0000 mon.a (mon.0) 226 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359415+0000 mon.a (mon.0) 227 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359525+0000 mon.a (mon.0) 228 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359574+0000 mon.a (mon.0) 229 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359609+0000 mon.a (mon.0) 230 : cluster 4 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:18:54.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359638+0000 mon.a (mon.0) 231 : cluster 4 application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:18:54.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.359667+0000 mon.a (mon.0) 232 : cluster 4 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:18:54.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:54 smithi148 bash[82222]: cluster 2024-03-07T03:18:53.370950+0000 mon.a (mon.0) 233 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2024-03-07T03:18:54.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:47.278315+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-03-07T03:18:54.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359145+0000 mon.a (mon.0) 220 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable; 1 pool(s) do not have an application enabled 2024-03-07T03:18:54.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359189+0000 mon.a (mon.0) 221 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359209+0000 mon.a (mon.0) 222 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359230+0000 mon.a (mon.0) 223 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359280+0000 mon.a (mon.0) 224 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359336+0000 mon.a (mon.0) 225 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359380+0000 mon.a (mon.0) 226 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359415+0000 mon.a (mon.0) 227 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359525+0000 mon.a (mon.0) 228 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359574+0000 mon.a (mon.0) 229 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359609+0000 mon.a (mon.0) 230 : cluster 4 [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359638+0000 mon.a (mon.0) 231 : cluster 4 application not enabled on pool 'test-librbd-smithi148-68562-9' 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.359667+0000 mon.a (mon.0) 232 : cluster 4 use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-03-07T03:18:54.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:54 smithi195 bash[23221]: cluster 2024-03-07T03:18:53.370950+0000 mon.a (mon.0) 233 : cluster 0 osdmap e365: 8 total, 8 up, 8 in 2024-03-07T03:18:55.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:55 smithi148 bash[79819]: cluster 2024-03-07T03:18:54.230592+0000 mgr.y (mgr.34101) 72 : cluster 0 pgmap v133: 105 pgs: 105 active+clean; 2.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:18:55.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:55 smithi148 bash[79819]: cluster 2024-03-07T03:18:54.362320+0000 mon.a (mon.0) 234 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:18:55.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:55 smithi148 bash[79819]: cluster 2024-03-07T03:18:54.389232+0000 mon.a (mon.0) 235 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2024-03-07T03:18:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:55 smithi148 bash[82222]: cluster 2024-03-07T03:18:54.230592+0000 mgr.y (mgr.34101) 72 : cluster 0 pgmap v133: 105 pgs: 105 active+clean; 2.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:18:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:55 smithi148 bash[82222]: cluster 2024-03-07T03:18:54.362320+0000 mon.a (mon.0) 234 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:18:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:55 smithi148 bash[82222]: cluster 2024-03-07T03:18:54.389232+0000 mon.a (mon.0) 235 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2024-03-07T03:18:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:55 smithi195 bash[23221]: cluster 2024-03-07T03:18:54.230592+0000 mgr.y (mgr.34101) 72 : cluster 0 pgmap v133: 105 pgs: 105 active+clean; 2.2 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:18:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:55 smithi195 bash[23221]: cluster 2024-03-07T03:18:54.362320+0000 mon.a (mon.0) 234 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:18:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:55 smithi195 bash[23221]: cluster 2024-03-07T03:18:54.389232+0000 mon.a (mon.0) 235 : cluster 0 osdmap e366: 8 total, 8 up, 8 in 2024-03-07T03:18:56.375 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestGroup.group_create 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[79819]: cluster 2024-03-07T03:18:55.383840+0000 mon.a (mon.0) 236 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[79819]: audit 2024-03-07T03:18:55.387780+0000 mon.a (mon.0) 237 : audit 1 from='client.? 172.21.15.148:0/230713821' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[79819]: audit 2024-03-07T03:18:56.378907+0000 mon.a (mon.0) 238 : audit 1 from='client.? 172.21.15.148:0/230713821' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[79819]: cluster 2024-03-07T03:18:56.387724+0000 mon.a (mon.0) 239 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[82222]: cluster 2024-03-07T03:18:55.383840+0000 mon.a (mon.0) 236 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[82222]: audit 2024-03-07T03:18:55.387780+0000 mon.a (mon.0) 237 : audit 1 from='client.? 172.21.15.148:0/230713821' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[82222]: audit 2024-03-07T03:18:56.378907+0000 mon.a (mon.0) 238 : audit 1 from='client.? 172.21.15.148:0/230713821' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:18:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:56 smithi148 bash[82222]: cluster 2024-03-07T03:18:56.387724+0000 mon.a (mon.0) 239 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2024-03-07T03:18:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:56 smithi195 bash[23221]: cluster 2024-03-07T03:18:55.383840+0000 mon.a (mon.0) 236 : cluster 0 osdmap e367: 8 total, 8 up, 8 in 2024-03-07T03:18:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:56 smithi195 bash[23221]: audit 2024-03-07T03:18:55.387780+0000 mon.a (mon.0) 237 : audit 1 from='client.? 172.21.15.148:0/230713821' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-10","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:18:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:56 smithi195 bash[23221]: audit 2024-03-07T03:18:56.378907+0000 mon.a (mon.0) 238 : audit 1 from='client.? 172.21.15.148:0/230713821' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:18:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:56 smithi195 bash[23221]: cluster 2024-03-07T03:18:56.387724+0000 mon.a (mon.0) 239 : cluster 0 osdmap e368: 8 total, 8 up, 8 in 2024-03-07T03:18:57.069 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 2 2024-03-07T03:18:57.401 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[72584]: debug 2024-03-07T03:18:57.073+0000 7f6cc280d700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-03-07T03:18:57.401 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:18:57] "GET /metrics HTTP/1.1" 200 38833 "" "Prometheus/2.43.0" 2024-03-07T03:18:57.765 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[79819]: cluster 2024-03-07T03:18:56.231402+0000 mgr.y (mgr.34101) 73 : cluster 0 pgmap v136: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:18:57.765 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[79819]: cluster 2024-03-07T03:18:56.388802+0000 mon.a (mon.0) 240 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:18:57.765 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[79819]: audit 2024-03-07T03:18:57.065407+0000 mon.a (mon.0) 241 : audit 0 from='client.? 172.21.15.148:0/1422458143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:18:57.766 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[79819]: audit 2024-03-07T03:18:57.298777+0000 mon.a (mon.0) 242 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:57.766 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[79819]: cluster 2024-03-07T03:18:57.405754+0000 mon.a (mon.0) 243 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2024-03-07T03:18:57.766 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[82222]: cluster 2024-03-07T03:18:56.231402+0000 mgr.y (mgr.34101) 73 : cluster 0 pgmap v136: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:18:57.766 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[82222]: cluster 2024-03-07T03:18:56.388802+0000 mon.a (mon.0) 240 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:18:57.766 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[82222]: audit 2024-03-07T03:18:57.065407+0000 mon.a (mon.0) 241 : audit 0 from='client.? 172.21.15.148:0/1422458143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:18:57.766 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[82222]: audit 2024-03-07T03:18:57.298777+0000 mon.a (mon.0) 242 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:57.766 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:57 smithi148 bash[82222]: cluster 2024-03-07T03:18:57.405754+0000 mon.a (mon.0) 243 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2024-03-07T03:18:57.767 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:18:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:57 smithi195 bash[23221]: cluster 2024-03-07T03:18:56.231402+0000 mgr.y (mgr.34101) 73 : cluster 0 pgmap v136: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.6 GiB used, 710 GiB / 715 GiB avail 2024-03-07T03:18:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:57 smithi195 bash[23221]: cluster 2024-03-07T03:18:56.388802+0000 mon.a (mon.0) 240 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:18:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:57 smithi195 bash[23221]: audit 2024-03-07T03:18:57.065407+0000 mon.a (mon.0) 241 : audit 0 from='client.? 172.21.15.148:0/1422458143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:18:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:57 smithi195 bash[23221]: audit 2024-03-07T03:18:57.298777+0000 mon.a (mon.0) 242 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:18:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:57 smithi195 bash[23221]: cluster 2024-03-07T03:18:57.405754+0000 mon.a (mon.0) 243 : cluster 0 osdmap e369: 8 total, 8 up, 8 in 2024-03-07T03:18:58.440 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestGroup.group_create (2065 ms) 2024-03-07T03:18:58.440 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestGroup.group_createPP 2024-03-07T03:18:58.469 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestGroup.group_createPP (29 ms) 2024-03-07T03:18:58.469 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestGroup.add_image 2024-03-07T03:18:58.527 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestGroup.add_image (58 ms) 2024-03-07T03:18:58.527 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestGroup.add_imagePP 2024-03-07T03:18:58.596 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestGroup.add_imagePP (69 ms) 2024-03-07T03:18:58.596 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestGroup.add_snapshot 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[79819]: audit 2024-03-07T03:18:57.754275+0000 mon.a (mon.0) 244 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[79819]: audit 2024-03-07T03:18:57.765764+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[79819]: cluster 2024-03-07T03:18:58.232548+0000 mgr.y (mgr.34101) 74 : cluster 0 pgmap v139: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[79819]: cluster 2024-03-07T03:18:58.409629+0000 mon.a (mon.0) 246 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[82222]: audit 2024-03-07T03:18:57.754275+0000 mon.a (mon.0) 244 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[82222]: audit 2024-03-07T03:18:57.765764+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[82222]: cluster 2024-03-07T03:18:58.232548+0000 mgr.y (mgr.34101) 74 : cluster 0 pgmap v139: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:58 smithi148 bash[82222]: cluster 2024-03-07T03:18:58.409629+0000 mon.a (mon.0) 246 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2024-03-07T03:18:59.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:58 smithi195 bash[23221]: audit 2024-03-07T03:18:57.754275+0000 mon.a (mon.0) 244 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:59.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:58 smithi195 bash[23221]: audit 2024-03-07T03:18:57.765764+0000 mon.a (mon.0) 245 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:18:59.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:58 smithi195 bash[23221]: cluster 2024-03-07T03:18:58.232548+0000 mgr.y (mgr.34101) 74 : cluster 0 pgmap v139: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:18:59.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:58 smithi195 bash[23221]: cluster 2024-03-07T03:18:58.409629+0000 mon.a (mon.0) 246 : cluster 0 osdmap e370: 8 total, 8 up, 8 in 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[79819]: cluster 2024-03-07T03:18:58.763490+0000 mon.a (mon.0) 247 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[79819]: audit 2024-03-07T03:18:58.865245+0000 mon.a (mon.0) 248 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[79819]: audit 2024-03-07T03:18:58.870914+0000 mon.a (mon.0) 249 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[79819]: cluster 2024-03-07T03:18:59.412037+0000 mon.a (mon.0) 250 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[82222]: cluster 2024-03-07T03:18:58.763490+0000 mon.a (mon.0) 247 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[82222]: audit 2024-03-07T03:18:58.865245+0000 mon.a (mon.0) 248 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[82222]: audit 2024-03-07T03:18:58.870914+0000 mon.a (mon.0) 249 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:00.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:18:59 smithi148 bash[82222]: cluster 2024-03-07T03:18:59.412037+0000 mon.a (mon.0) 250 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2024-03-07T03:19:00.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:59 smithi195 bash[23221]: cluster 2024-03-07T03:18:58.763490+0000 mon.a (mon.0) 247 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:00.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:59 smithi195 bash[23221]: audit 2024-03-07T03:18:58.865245+0000 mon.a (mon.0) 248 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:00.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:59 smithi195 bash[23221]: audit 2024-03-07T03:18:58.870914+0000 mon.a (mon.0) 249 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:00.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:18:59 smithi195 bash[23221]: cluster 2024-03-07T03:18:59.412037+0000 mon.a (mon.0) 250 : cluster 0 osdmap e371: 8 total, 8 up, 8 in 2024-03-07T03:19:01.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:01 smithi148 bash[79819]: cluster 2024-03-07T03:19:00.233179+0000 mgr.y (mgr.34101) 75 : cluster 0 pgmap v142: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:19:01.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:01 smithi148 bash[79819]: cluster 2024-03-07T03:19:00.415004+0000 mon.a (mon.0) 251 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2024-03-07T03:19:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:01 smithi148 bash[82222]: cluster 2024-03-07T03:19:00.233179+0000 mgr.y (mgr.34101) 75 : cluster 0 pgmap v142: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:19:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:01 smithi148 bash[82222]: cluster 2024-03-07T03:19:00.415004+0000 mon.a (mon.0) 251 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2024-03-07T03:19:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:01 smithi195 bash[23221]: cluster 2024-03-07T03:19:00.233179+0000 mgr.y (mgr.34101) 75 : cluster 0 pgmap v142: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-07T03:19:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:01 smithi195 bash[23221]: cluster 2024-03-07T03:19:00.415004+0000 mon.a (mon.0) 251 : cluster 0 osdmap e372: 8 total, 8 up, 8 in 2024-03-07T03:19:02.477 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:19:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:02 smithi195 bash[23221]: cluster 2024-03-07T03:19:01.428197+0000 mon.a (mon.0) 252 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2024-03-07T03:19:02.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:02 smithi195 bash[59134]: ts=2024-03-07T03:19:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:02.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:02 smithi148 bash[79819]: cluster 2024-03-07T03:19:01.428197+0000 mon.a (mon.0) 252 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2024-03-07T03:19:02.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:02 smithi148 bash[82222]: cluster 2024-03-07T03:19:01.428197+0000 mon.a (mon.0) 252 : cluster 0 osdmap e373: 8 total, 8 up, 8 in 2024-03-07T03:19:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:03 smithi148 bash[82222]: cluster 2024-03-07T03:19:02.233904+0000 mgr.y (mgr.34101) 76 : cluster 0 pgmap v145: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:03 smithi148 bash[82222]: cluster 2024-03-07T03:19:02.438897+0000 mon.a (mon.0) 253 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2024-03-07T03:19:03.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:03 smithi148 bash[79819]: cluster 2024-03-07T03:19:02.233904+0000 mgr.y (mgr.34101) 76 : cluster 0 pgmap v145: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:03.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:03 smithi148 bash[79819]: cluster 2024-03-07T03:19:02.438897+0000 mon.a (mon.0) 253 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2024-03-07T03:19:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:03 smithi195 bash[23221]: cluster 2024-03-07T03:19:02.233904+0000 mgr.y (mgr.34101) 76 : cluster 0 pgmap v145: 105 pgs: 105 active+clean; 582 KiB data, 352 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:03.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:03 smithi195 bash[23221]: cluster 2024-03-07T03:19:02.438897+0000 mon.a (mon.0) 253 : cluster 0 osdmap e374: 8 total, 8 up, 8 in 2024-03-07T03:19:03.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:03 smithi195 bash[59134]: ts=2024-03-07T03:19:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:04.493 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestGroup.add_snapshot (5897 ms) 2024-03-07T03:19:04.493 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-03-07T03:19:04.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:04 smithi148 bash[79819]: cluster 2024-03-07T03:19:03.444274+0000 mon.a (mon.0) 254 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2024-03-07T03:19:04.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:04 smithi148 bash[82222]: cluster 2024-03-07T03:19:03.444274+0000 mon.a (mon.0) 254 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2024-03-07T03:19:04.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:04 smithi195 bash[23221]: cluster 2024-03-07T03:19:03.444274+0000 mon.a (mon.0) 254 : cluster 0 osdmap e375: 8 total, 8 up, 8 in 2024-03-07T03:19:05.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:05 smithi148 bash[79819]: cluster 2024-03-07T03:19:04.234595+0000 mgr.y (mgr.34101) 77 : cluster 0 pgmap v148: 105 pgs: 105 active+clean; 582 KiB data, 357 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:19:05.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:05 smithi148 bash[79819]: cluster 2024-03-07T03:19:04.457314+0000 mon.a (mon.0) 255 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2024-03-07T03:19:05.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:05 smithi148 bash[82222]: cluster 2024-03-07T03:19:04.234595+0000 mgr.y (mgr.34101) 77 : cluster 0 pgmap v148: 105 pgs: 105 active+clean; 582 KiB data, 357 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:19:05.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:05 smithi148 bash[82222]: cluster 2024-03-07T03:19:04.457314+0000 mon.a (mon.0) 255 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2024-03-07T03:19:05.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:05 smithi195 bash[23221]: cluster 2024-03-07T03:19:04.234595+0000 mgr.y (mgr.34101) 77 : cluster 0 pgmap v148: 105 pgs: 105 active+clean; 582 KiB data, 357 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:19:05.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:05 smithi195 bash[23221]: cluster 2024-03-07T03:19:04.457314+0000 mon.a (mon.0) 255 : cluster 0 osdmap e376: 8 total, 8 up, 8 in 2024-03-07T03:19:06.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:06 smithi148 bash[79819]: cluster 2024-03-07T03:19:05.461562+0000 mon.a (mon.0) 256 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2024-03-07T03:19:06.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:06 smithi148 bash[82222]: cluster 2024-03-07T03:19:05.461562+0000 mon.a (mon.0) 256 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2024-03-07T03:19:06.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:06 smithi195 bash[23221]: cluster 2024-03-07T03:19:05.461562+0000 mon.a (mon.0) 256 : cluster 0 osdmap e377: 8 total, 8 up, 8 in 2024-03-07T03:19:07.271 INFO:teuthology.orchestra.run.smithi148.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:19:07.271 INFO:teuthology.orchestra.run.smithi148.stdout:alertmanager.a smithi148 *:9093,9094 running (9m) 9s ago 9m 14.2M - 0.25.0 c8568f914cd2 a6ee726aca40 2024-03-07T03:19:07.271 INFO:teuthology.orchestra.run.smithi148.stdout:grafana.a smithi195 *:3000 running (9m) 30s ago 9m 79.6M - 9.4.7 954c08fa6188 a09a8cb7fec4 2024-03-07T03:19:07.271 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi148.qizikc smithi148 running (7m) 9s ago 7m 14.8M - 18.2.1-718-gae75fc9d 0164bb8a7c67 d82cf82f7898 2024-03-07T03:19:07.271 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi195.gtlmjn smithi195 running (7m) 30s ago 7m 17.0M - 18.2.1-718-gae75fc9d 0164bb8a7c67 f9ba5c67a194 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.x smithi195 *:8443,9283,8765 running (2m) 30s ago 17m 437M - 19.0.0-1905-gea9a5688 fdb83767c1ab 758d97044603 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.y smithi148 *:8443,9283,8765 running (3m) 9s ago 20m 491M - 19.0.0-1905-gea9a5688 fdb83767c1ab 85cf9ffcb609 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:mon.a smithi148 running (2m) 9s ago 20m 45.8M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 39441e18edee 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:mon.b smithi195 running (18m) 30s ago 18m 56.6M 2048M 18.2.1-718-gae75fc9d 0164bb8a7c67 aa6d6fcf0602 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:mon.c smithi148 running (21s) 9s ago 18m 18.1M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab a8f98b5ace69 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.a smithi148 *:9100 running (10m) 9s ago 10m 8311k - 1.5.0 0da6a335fe13 d9b895f99d5c 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.b smithi195 *:9100 running (10m) 30s ago 10m 8859k - 1.5.0 0da6a335fe13 be49ab11d18e 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.0 smithi148 running (17m) 9s ago 17m 171M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 f8c58575ec57 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.1 smithi148 running (16m) 9s ago 16m 139M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 3125450ae62f 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.2 smithi148 running (15m) 9s ago 15m 129M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 b463a715bf36 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.3 smithi148 running (14m) 9s ago 14m 135M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 180f02eefffa 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.4 smithi195 running (13m) 30s ago 13m 138M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 5023ce93366a 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.5 smithi195 running (13m) 30s ago 13m 94.4M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c850d7f8ae40 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.6 smithi195 running (12m) 30s ago 12m 167M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 332f73f59fdc 2024-03-07T03:19:07.272 INFO:teuthology.orchestra.run.smithi148.stdout:osd.7 smithi195 running (11m) 30s ago 11m 118M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9a7e2329950b 2024-03-07T03:19:07.273 INFO:teuthology.orchestra.run.smithi148.stdout:prometheus.a smithi195 *:9095 running (2m) 30s ago 10m 46.0M - 2.43.0 a07b618ecd1d e5c6727b35a4 2024-03-07T03:19:07.462 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:19:07] "GET /metrics HTTP/1.1" 200 36331 "" "Prometheus/2.43.0" 2024-03-07T03:19:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[79819]: cluster 2024-03-07T03:19:06.235238+0000 mgr.y (mgr.34101) 78 : cluster 0 pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 357 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:19:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[79819]: cluster 2024-03-07T03:19:06.482264+0000 mon.a (mon.0) 257 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2024-03-07T03:19:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[79819]: cluster 2024-03-07T03:19:07.032087+0000 mon.a (mon.0) 258 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2024-03-07T03:19:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[79819]: audit 2024-03-07T03:19:07.297882+0000 mon.a (mon.0) 259 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:07.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[82222]: cluster 2024-03-07T03:19:06.235238+0000 mgr.y (mgr.34101) 78 : cluster 0 pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 357 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:19:07.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[82222]: cluster 2024-03-07T03:19:06.482264+0000 mon.a (mon.0) 257 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2024-03-07T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[82222]: cluster 2024-03-07T03:19:07.032087+0000 mon.a (mon.0) 258 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2024-03-07T03:19:07.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:07 smithi148 bash[82222]: audit 2024-03-07T03:19:07.297882+0000 mon.a (mon.0) 259 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:07 smithi195 bash[23221]: cluster 2024-03-07T03:19:06.235238+0000 mgr.y (mgr.34101) 78 : cluster 0 pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 357 MiB used, 715 GiB / 715 GiB avail; 163 KiB/s rd, 18 KiB/s wr, 220 op/s 2024-03-07T03:19:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:07 smithi195 bash[23221]: cluster 2024-03-07T03:19:06.482264+0000 mon.a (mon.0) 257 : cluster 0 osdmap e378: 8 total, 8 up, 8 in 2024-03-07T03:19:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:07 smithi195 bash[23221]: cluster 2024-03-07T03:19:07.032087+0000 mon.a (mon.0) 258 : cluster 0 osdmap e379: 8 total, 8 up, 8 in 2024-03-07T03:19:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:07 smithi195 bash[23221]: audit 2024-03-07T03:19:07.297882+0000 mon.a (mon.0) 259 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:08.460 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:19:08.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:08 smithi148 bash[82222]: audit 2024-03-07T03:19:07.254290+0000 mgr.y (mgr.34101) 79 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:08.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:08 smithi148 bash[82222]: cluster 2024-03-07T03:19:08.038594+0000 mon.a (mon.0) 260 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2024-03-07T03:19:08.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:08 smithi148 bash[79819]: audit 2024-03-07T03:19:07.254290+0000 mgr.y (mgr.34101) 79 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:08.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:08 smithi148 bash[79819]: cluster 2024-03-07T03:19:08.038594+0000 mon.a (mon.0) 260 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2024-03-07T03:19:08.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:08 smithi195 bash[23221]: audit 2024-03-07T03:19:07.254290+0000 mgr.y (mgr.34101) 79 : audit 0 from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:19:08.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:08 smithi195 bash[23221]: cluster 2024-03-07T03:19:08.038594+0000 mon.a (mon.0) 260 : cluster 0 osdmap e380: 8 total, 8 up, 8 in 2024-03-07T03:19:09.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:09 smithi148 bash[79819]: cluster 2024-03-07T03:19:08.236483+0000 mgr.y (mgr.34101) 80 : cluster 0 pgmap v155: 105 pgs: 105 active+clean; 583 KiB data, 379 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-03-07T03:19:09.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:09 smithi148 bash[79819]: cluster 2024-03-07T03:19:09.049644+0000 mon.a (mon.0) 261 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2024-03-07T03:19:09.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:09 smithi148 bash[82222]: cluster 2024-03-07T03:19:08.236483+0000 mgr.y (mgr.34101) 80 : cluster 0 pgmap v155: 105 pgs: 105 active+clean; 583 KiB data, 379 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-03-07T03:19:09.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:09 smithi148 bash[82222]: cluster 2024-03-07T03:19:09.049644+0000 mon.a (mon.0) 261 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2024-03-07T03:19:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:09 smithi195 bash[23221]: cluster 2024-03-07T03:19:08.236483+0000 mgr.y (mgr.34101) 80 : cluster 0 pgmap v155: 105 pgs: 105 active+clean; 583 KiB data, 379 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 13 KiB/s wr, 170 op/s 2024-03-07T03:19:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:09 smithi195 bash[23221]: cluster 2024-03-07T03:19:09.049644+0000 mon.a (mon.0) 261 : cluster 0 osdmap e381: 8 total, 8 up, 8 in 2024-03-07T03:19:10.094 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestGroup.add_snapshotPP (5601 ms) 2024-03-07T03:19:11.059 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 6 tests from TestGroup (13719 ms total) 2024-03-07T03:19:11.059 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:19:11.059 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 31 tests from TestDeepCopy 2024-03-07T03:19:11.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: cluster 2024-03-07T03:19:10.048555+0000 mon.a (mon.0) 262 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2024-03-07T03:19:11.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: cluster 2024-03-07T03:19:10.237141+0000 mgr.y (mgr.34101) 81 : cluster 0 pgmap v158: 105 pgs: 105 active+clean; 583 KiB data, 379 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 171 op/s 2024-03-07T03:19:11.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.528082+0000 mon.a (mon.0) 263 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.540101+0000 mon.a (mon.0) 264 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.542800+0000 mon.a (mon.0) 265 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:11.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.545192+0000 mon.a (mon.0) 266 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:11.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.555514+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.645492+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:11.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.649348+0000 mon.a (mon.0) 269 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:11.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.652647+0000 mon.a (mon.0) 270 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:19:11.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:11 smithi195 bash[23221]: audit 2024-03-07T03:19:10.654228+0000 mon.a (mon.0) 271 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-07T03:19:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: cluster 2024-03-07T03:19:10.048555+0000 mon.a (mon.0) 262 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2024-03-07T03:19:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: cluster 2024-03-07T03:19:10.237141+0000 mgr.y (mgr.34101) 81 : cluster 0 pgmap v158: 105 pgs: 105 active+clean; 583 KiB data, 379 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 171 op/s 2024-03-07T03:19:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.528082+0000 mon.a (mon.0) 263 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.540101+0000 mon.a (mon.0) 264 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.542800+0000 mon.a (mon.0) 265 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.545192+0000 mon.a (mon.0) 266 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.555514+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.645492+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.649348+0000 mon.a (mon.0) 269 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.652647+0000 mon.a (mon.0) 270 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[79819]: audit 2024-03-07T03:19:10.654228+0000 mon.a (mon.0) 271 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: cluster 2024-03-07T03:19:10.048555+0000 mon.a (mon.0) 262 : cluster 0 osdmap e382: 8 total, 8 up, 8 in 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: cluster 2024-03-07T03:19:10.237141+0000 mgr.y (mgr.34101) 81 : cluster 0 pgmap v158: 105 pgs: 105 active+clean; 583 KiB data, 379 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 13 KiB/s wr, 171 op/s 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.528082+0000 mon.a (mon.0) 263 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.540101+0000 mon.a (mon.0) 264 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.542800+0000 mon.a (mon.0) 265 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.545192+0000 mon.a (mon.0) 266 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.555514+0000 mon.a (mon.0) 267 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.645492+0000 mon.a (mon.0) 268 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.649348+0000 mon.a (mon.0) 269 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:19:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.652647+0000 mon.a (mon.0) 270 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-07T03:19:11.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:11 smithi148 bash[82222]: audit 2024-03-07T03:19:10.654228+0000 mon.a (mon.0) 271 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-07T03:19:12.359 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:12 smithi195 bash[23221]: cephadm 2024-03-07T03:19:10.655641+0000 mgr.y (mgr.34101) 82 : cephadm 1 Upgrade: It appears safe to stop mon.b 2024-03-07T03:19:12.359 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:12 smithi195 bash[23221]: cluster 2024-03-07T03:19:11.066501+0000 mon.a (mon.0) 272 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2024-03-07T03:19:12.359 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:12 smithi195 bash[59134]: ts=2024-03-07T03:19:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:12.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:12 smithi148 bash[79819]: cephadm 2024-03-07T03:19:10.655641+0000 mgr.y (mgr.34101) 82 : cephadm 1 Upgrade: It appears safe to stop mon.b 2024-03-07T03:19:12.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:12 smithi148 bash[79819]: cluster 2024-03-07T03:19:11.066501+0000 mon.a (mon.0) 272 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2024-03-07T03:19:12.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:12 smithi148 bash[82222]: cephadm 2024-03-07T03:19:10.655641+0000 mgr.y (mgr.34101) 82 : cephadm 1 Upgrade: It appears safe to stop mon.b 2024-03-07T03:19:12.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:12 smithi148 bash[82222]: cluster 2024-03-07T03:19:11.066501+0000 mon.a (mon.0) 272 : cluster 0 osdmap e383: 8 total, 8 up, 8 in 2024-03-07T03:19:13.060 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Empty 2024-03-07T03:19:13.196 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:19:13.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: cluster 2024-03-07T03:19:12.064114+0000 mon.a (mon.0) 273 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2024-03-07T03:19:13.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: audit 2024-03-07T03:19:12.073150+0000 mon.c (mon.2) 2 : audit 1 from='client.? 172.21.15.148:0/3544809895' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:13.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: audit 2024-03-07T03:19:12.084301+0000 mon.a (mon.0) 274 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:13.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: cluster 2024-03-07T03:19:12.237865+0000 mgr.y (mgr.34101) 83 : cluster 0 pgmap v161: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 379 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:13.495 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: audit 2024-03-07T03:19:12.307778+0000 mon.a (mon.0) 275 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:13.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: audit 2024-03-07T03:19:12.310021+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:13.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: audit 2024-03-07T03:19:13.061304+0000 mon.a (mon.0) 277 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:19:13.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[23221]: cluster 2024-03-07T03:19:13.073069+0000 mon.a (mon.0) 278 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: cluster 2024-03-07T03:19:12.064114+0000 mon.a (mon.0) 273 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: audit 2024-03-07T03:19:12.073150+0000 mon.c (mon.2) 2 : audit 1 from='client.? 172.21.15.148:0/3544809895' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: audit 2024-03-07T03:19:12.084301+0000 mon.a (mon.0) 274 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: cluster 2024-03-07T03:19:12.237865+0000 mgr.y (mgr.34101) 83 : cluster 0 pgmap v161: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 379 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: audit 2024-03-07T03:19:12.307778+0000 mon.a (mon.0) 275 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: audit 2024-03-07T03:19:12.310021+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: audit 2024-03-07T03:19:13.061304+0000 mon.a (mon.0) 277 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:19:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[79819]: cluster 2024-03-07T03:19:13.073069+0000 mon.a (mon.0) 278 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: cluster 2024-03-07T03:19:12.064114+0000 mon.a (mon.0) 273 : cluster 0 osdmap e384: 8 total, 8 up, 8 in 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: audit 2024-03-07T03:19:12.073150+0000 mon.c (mon.2) 2 : audit 1 from='client.? 172.21.15.148:0/3544809895' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: audit 2024-03-07T03:19:12.084301+0000 mon.a (mon.0) 274 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: cluster 2024-03-07T03:19:12.237865+0000 mgr.y (mgr.34101) 83 : cluster 0 pgmap v161: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 379 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: audit 2024-03-07T03:19:12.307778+0000 mon.a (mon.0) 275 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: audit 2024-03-07T03:19:12.310021+0000 mon.a (mon.0) 276 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: audit 2024-03-07T03:19:13.061304+0000 mon.a (mon.0) 277 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:19:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:13 smithi148 bash[82222]: cluster 2024-03-07T03:19:13.073069+0000 mon.a (mon.0) 278 : cluster 0 osdmap e385: 8 total, 8 up, 8 in 2024-03-07T03:19:13.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:13 smithi195 bash[59134]: ts=2024-03-07T03:19:13.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:14.409 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: cluster 2024-03-07T03:19:13.077030+0000 mon.a (mon.0) 279 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:19:14.409 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: cephadm 2024-03-07T03:19:13.181676+0000 mgr.y (mgr.34101) 84 : cephadm 1 Upgrade: Updating mon.b 2024-03-07T03:19:14.409 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: audit 2024-03-07T03:19:13.187308+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:14.409 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: audit 2024-03-07T03:19:13.191411+0000 mon.a (mon.0) 281 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:14.409 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: audit 2024-03-07T03:19:13.193314+0000 mon.a (mon.0) 282 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:14.409 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: audit 2024-03-07T03:19:13.194929+0000 mon.a (mon.0) 283 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: cephadm 2024-03-07T03:19:13.196866+0000 mgr.y (mgr.34101) 85 : cephadm 1 Deploying daemon mon.b on smithi195 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[79819]: cluster 2024-03-07T03:19:14.095708+0000 mon.a (mon.0) 284 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: cluster 2024-03-07T03:19:13.077030+0000 mon.a (mon.0) 279 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: cephadm 2024-03-07T03:19:13.181676+0000 mgr.y (mgr.34101) 84 : cephadm 1 Upgrade: Updating mon.b 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: audit 2024-03-07T03:19:13.187308+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: audit 2024-03-07T03:19:13.191411+0000 mon.a (mon.0) 281 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: audit 2024-03-07T03:19:13.193314+0000 mon.a (mon.0) 282 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: audit 2024-03-07T03:19:13.194929+0000 mon.a (mon.0) 283 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: cephadm 2024-03-07T03:19:13.196866+0000 mgr.y (mgr.34101) 85 : cephadm 1 Deploying daemon mon.b on smithi195 2024-03-07T03:19:14.410 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:14 smithi148 bash[82222]: cluster 2024-03-07T03:19:14.095708+0000 mon.a (mon.0) 284 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2024-03-07T03:19:14.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: cluster 2024-03-07T03:19:13.077030+0000 mon.a (mon.0) 279 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:19:14.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: cephadm 2024-03-07T03:19:13.181676+0000 mgr.y (mgr.34101) 84 : cephadm 1 Upgrade: Updating mon.b 2024-03-07T03:19:14.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: audit 2024-03-07T03:19:13.187308+0000 mon.a (mon.0) 280 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:14.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: audit 2024-03-07T03:19:13.191411+0000 mon.a (mon.0) 281 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:14.539 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: audit 2024-03-07T03:19:13.193314+0000 mon.a (mon.0) 282 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:14.540 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: audit 2024-03-07T03:19:13.194929+0000 mon.a (mon.0) 283 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:14.540 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: cephadm 2024-03-07T03:19:13.196866+0000 mgr.y (mgr.34101) 85 : cephadm 1 Deploying daemon mon.b on smithi195 2024-03-07T03:19:14.540 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:14 smithi195 bash[23221]: cluster 2024-03-07T03:19:14.095708+0000 mon.a (mon.0) 284 : cluster 0 osdmap e386: 8 total, 8 up, 8 in 2024-03-07T03:19:15.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:15 smithi148 bash[79819]: cluster 2024-03-07T03:19:14.238514+0000 mgr.y (mgr.34101) 86 : cluster 0 pgmap v164: 105 pgs: 105 active+clean; 582 KiB data, 380 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:15.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:15 smithi148 bash[82222]: cluster 2024-03-07T03:19:14.238514+0000 mgr.y (mgr.34101) 86 : cluster 0 pgmap v164: 105 pgs: 105 active+clean; 582 KiB data, 380 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:15.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:15 smithi195 bash[23221]: cluster 2024-03-07T03:19:14.238514+0000 mgr.y (mgr.34101) 86 : cluster 0 pgmap v164: 105 pgs: 105 active+clean; 582 KiB data, 380 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:15.336 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Empty (2277 ms) 2024-03-07T03:19:15.336 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-03-07T03:19:15.681 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.NoSnaps (345 ms) 2024-03-07T03:19:15.682 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Snaps 2024-03-07T03:19:15.789 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.790 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.790 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.790 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.790 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.791 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.791 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.792 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:15.792 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:16.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:15 smithi195 systemd[1]: Stopping Ceph mon.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:19:16.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:15 smithi195 bash[23221]: debug 2024-03-07T03:19:15.912+0000 7fca4fb3a700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:19:16.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:15 smithi195 bash[23221]: debug 2024-03-07T03:19:15.912+0000 7fca4fb3a700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-03-07T03:19:17.051 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:16 smithi195 bash[66425]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-b 2024-03-07T03:19:17.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:16 smithi195 bash[66487]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-b 2024-03-07T03:19:17.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:16 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.b.service: Deactivated successfully. 2024-03-07T03:19:17.052 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:16 smithi195 systemd[1]: Stopped Ceph mon.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:19:17.312 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.312 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: Started Ceph mon.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:19:17.313 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.313 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.313 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.314 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.314 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.314 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:19:17 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:19:17.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:19:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:19:17] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.43.0" 2024-03-07T03:19:18.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.364+0000 7f635d79ac80 0 set uid:gid to 167:167 (ceph:ceph) 2024-03-07T03:19:18.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.364+0000 7f635d79ac80 0 ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev), process ceph-mon, pid 7 2024-03-07T03:19:18.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.364+0000 7f635d79ac80 0 pidfile_write: ignore empty --pid-file 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 0 load: jerasure load: lrc 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: RocksDB version: 7.9.2 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Git sha 0 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Compile date 2024-03-06 19:59:24 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: DB SUMMARY 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: DB Session ID: 6JWPOO3X3ZM4HKBN07ME 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: CURRENT file: CURRENT 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: IDENTITY file: IDENTITY 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 2628 Bytes 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000042.sst 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000040.log size: 2425545 ; 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.error_if_exists: 0 2024-03-07T03:19:18.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.create_if_missing: 0 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.paranoid_checks: 1 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.env: 0x55bf0eb5bb20 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.fs: PosixFileSystem 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.info_log: 0x55bf0f8ab920 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_file_opening_threads: 16 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.statistics: (nil) 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.use_fsync: 0 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_log_file_size: 0 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-07T03:19:18.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.keep_log_file_num: 1000 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.recycle_log_file_num: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.allow_fallocate: 1 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.allow_mmap_reads: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.allow_mmap_writes: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.use_direct_reads: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.create_missing_column_families: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.db_log_dir: 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.wal_dir: 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-03-07T03:19:18.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.advise_random_on_open: 1 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.db_write_buffer_size: 0 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.write_buffer_manager: 0x55bf0f87edc0 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.rate_limiter: (nil) 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-07T03:19:18.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.wal_recovery_mode: 2 2024-03-07T03:19:18.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.enable_thread_tracking: 0 2024-03-07T03:19:18.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.enable_pipelined_write: 0 2024-03-07T03:19:18.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.unordered_write: 0 2024-03-07T03:19:18.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.row_cache: None 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.wal_filter: None 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.allow_ingest_behind: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.two_write_queues: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.manual_wal_flush: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.wal_compression: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.atomic_flush: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-03-07T03:19:18.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.log_readahead_size: 0 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.best_efforts_recovery: 0 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.allow_data_in_errors: 0 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.db_host_id: __hostname__ 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_background_jobs: 2 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_background_compactions: -1 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_subcompactions: 1 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-07T03:19:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_total_wal_size: 0 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_open_files: -1 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bytes_per_sync: 0 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_readahead_size: 0 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_background_flushes: -1 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Compression algorithms supported: 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kZSTD supported: 0 2024-03-07T03:19:18.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kXpressCompression supported: 0 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kLZ4HCCompression supported: 1 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kZlibCompression supported: 1 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kSnappyCompression supported: 1 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kLZ4Compression supported: 1 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: kBZip2Compression supported: 0 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.merge_operator: 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_filter: None 2024-03-07T03:19:18.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_filter_factory: None 2024-03-07T03:19:18.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.sst_partitioner_factory: None 2024-03-07T03:19:18.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-03-07T03:19:18.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-03-07T03:19:18.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bf0f8aba20) 2024-03-07T03:19:18.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: cache_index_and_filter_blocks: 1 2024-03-07T03:19:18.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: pin_top_level_index_and_filter: 1 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: index_type: 0 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: data_block_index_type: 0 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: index_shortening: 1 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: data_block_hash_table_util_ratio: 0.750000 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: checksum: 4 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: no_block_cache: 0 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_cache: 0x55bf0f8af090 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_cache_name: BinnedLRUCache 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_cache_options: 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: capacity : 536870912 2024-03-07T03:19:18.576 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: num_shard_bits : 4 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: strict_capacity_limit : 0 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: high_pri_pool_ratio: 0.000 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_cache_compressed: (nil) 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: persistent_cache: (nil) 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_size: 4096 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_size_deviation: 10 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_restart_interval: 16 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: index_block_restart_interval: 1 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: metadata_block_size: 4096 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: partition_filters: 0 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: use_delta_encoding: 1 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: filter_policy: bloomfilter 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: whole_key_filtering: 1 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: verify_compression: 0 2024-03-07T03:19:18.577 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: read_amp_bytes_per_bit: 0 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: format_version: 5 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: enable_index_compression: 1 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: block_align: 0 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: max_auto_readahead_size: 262144 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: prepopulate_block_cache: 0 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: initial_auto_readahead_size: 8192 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: num_file_reads_for_auto_readahead: 2 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.write_buffer_size: 33554432 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_write_buffer_number: 2 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression: NoCompression 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression: Disabled 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.prefix_extractor: nullptr 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.num_levels: 7 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-07T03:19:18.578 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.level: 32767 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.strategy: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-07T03:19:18.579 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.enabled: false 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.target_file_size_base: 67108864 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-07T03:19:18.580 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.arena_block_size: 1048576 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.disable_auto_compactions: 0 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-07T03:19:18.581 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.table_properties_collectors: 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.inplace_update_support: 0 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-03-07T03:19:18.582 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.bloom_locality: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.max_successive_merges: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.paranoid_file_checks: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.force_consistency_checks: 1 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.report_bg_io_stats: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.ttl: 2592000 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.enable_blob_files: false 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.min_blob_size: 0 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.blob_file_size: 268435456 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-03-07T03:19:18.583 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.blob_file_starting_level: 0 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.368+0000 7f635d79ac80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.372+0000 7f635d79ac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.372+0000 7f635d79ac80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 44, last_sequence is 24437, log_number is 40,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 40 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.372+0000 7f635d79ac80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 40 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.372+0000 7f635d79ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: dc1c6b4e-2c62-436c-bc71-3d61b7c6d2af 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.372+0000 7f635d79ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781558372744, "job": 1, "event": "recovery_started", "wal_files": [40]} 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.372+0000 7f635d79ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #40 mode 2 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f635d79ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781558382585, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 45, "file_size": 1486171, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 24443, "largest_seqno": 25477, "table_properties": {"data_size": 1480814, "index_size": 3318, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1029, "raw_key_size": 9480, "raw_average_key_size": 23, "raw_value_size": 1471275, "raw_average_value_size": 3597, "num_data_blocks": 145, "num_entries": 409, "num_filter_entries": 409, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1709781558, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "dc1c6b4e-2c62-436c-bc71-3d61b7c6d2af", "db_session_id": "6JWPOO3X3ZM4HKBN07ME", "orig_file_number": 45, "seqno_to_time_mapping": "N/A"}} 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f635d79ac80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1709781558382715, "job": 1, "event": "recovery_finished"} 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f635d79ac80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 47 2024-03-07T03:19:18.584 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f635d79ac80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f635d79ac80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000040.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f635d79ac80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bf0f9a6000 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f635d79ac80 4 rocksdb: DB pointer 0x55bf0f990000 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f634bf03700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.380+0000 7f634bf03700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: ** DB Stats ** 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: ** Compaction Stats [default] ** 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:19:18.585 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: L0 1/0 1.42 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 330.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: L6 1/0 11.70 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Sum 2/0 13.11 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 330.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 330.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: ** Compaction Stats [default] ** 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 330.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-07T03:19:18.586 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Uptime(secs): 0.0 total, 0.0 interval 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Flush(GB): cumulative 0.001, interval 0.001 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: AddFile(Total Files): cumulative 0, interval 0 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: AddFile(L0 Files): cumulative 0, interval 0 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: AddFile(Keys): cumulative 0, interval 0 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Cumulative compaction: 0.00 GB write, 110.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Interval compaction: 0.00 GB write, 110.74 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Block cache BinnedLRUCache@0x55bf0f8af090#7 capacity: 512.00 MB usage: 4.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Block cache entry stats(count,size,portion): FilterBlock(1,1.16 KB,0.000220537%) IndexBlock(1,3.33 KB,0.000634789%) Misc(1,0.00 KB,0%) 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: ** File Read Latency Histogram By Level [default] ** 2024-03-07T03:19:18.587 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] at bind addrs [v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 1 mon.b@-1(???) e3 preinit fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 0 mon.b@-1(???).mds e10 new map 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 0 mon.b@-1(???).mds e10 print_map 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: e10 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: enable_multiple, ever_enabled_multiple: 1,1 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: legacy client fscid: 1 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Filesystem 'foo' (1) 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: fs_name foo 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: epoch 10 2024-03-07T03:19:18.588 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: created 2024-03-07T03:11:17.721445+0000 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: modified 2024-03-07T03:11:28.139230+0000 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: tableserver 0 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: root 0 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: session_timeout 60 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: session_autoclose 300 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: max_file_size 1099511627776 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: max_xattr_size 65536 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: required_client_features {} 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: last_failure 0 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: last_failure_osd_epoch 70 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-07T03:19:18.589 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: max_mds 1 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: in 0 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: up {0=24385} 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: failed 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: damaged 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: stopped 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: data_pools [4] 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: metadata_pool 3 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: inline_data disabled 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: balancer 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: bal_rank_mask -1 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: standby_count_wanted 1 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: [mds.foo.smithi195.gtlmjn{0:24385} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.195:6832/3075511892,v1:172.21.15.195:6833/3075511892] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: 2024-03-07T03:19:18.590 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: Standby daemons: 2024-03-07T03:19:18.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: 2024-03-07T03:19:18.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: [mds.foo.smithi148.qizikc{-1:14580} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.148:6834/4101454543,v1:172.21.15.148:6835/4101454543] compat {c=[1],r=[1],i=[7ff]}] 2024-03-07T03:19:18.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 0 mon.b@-1(???).osd e387 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-07T03:19:18.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 0 mon.b@-1(???).osd e387 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:19:18.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 0 mon.b@-1(???).osd e387 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:19:18.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 0 mon.b@-1(???).osd e387 crush map has features 432629239337189376, adjusting msgr requires 2024-03-07T03:19:18.591 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:18 smithi195 bash[66542]: debug 2024-03-07T03:19:18.384+0000 7f635d79ac80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2024-03-07T03:19:19.218 INFO:teuthology.orchestra.run.smithi148.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:19:19.533 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[82222]: cluster 2024-03-07T03:19:15.103358+0000 mon.a (mon.0) 285 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:19.533 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[82222]: cluster 2024-03-07T03:19:15.117927+0000 mon.a (mon.0) 286 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:19:19.533 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: cluster 2024-03-07T03:19:15.103358+0000 mon.a (mon.0) 285 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: cluster 2024-03-07T03:19:15.117927+0000 mon.a (mon.0) 286 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.160+0000 7f195103b700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:19:19.534 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.535 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:19 smithi148 bash[79819]: debug 2024-03-07T03:19:19.204+0000 7f1953840700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:19 smithi195 bash[66542]: cluster 2024-03-07T03:19:15.103358+0000 mon.a (mon.0) 285 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:19:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:19 smithi195 bash[66542]: cluster 2024-03-07T03:19:15.117927+0000 mon.a (mon.0) 286 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:19:20.103 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'sleep 60' 2024-03-07T03:19:20.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: audit 2024-03-07T03:19:19.195820+0000 mon.a (mon.0) 303 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.196173+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.196510+0000 mon.a (mon.0) 304 : cluster 1 mon.a calling monitor election 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.196630+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: audit 2024-03-07T03:19:19.198092+0000 mon.a (mon.0) 305 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: audit 2024-03-07T03:19:19.198258+0000 mon.a (mon.0) 306 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.200009+0000 mon.a (mon.0) 307 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.209781+0000 mon.a (mon.0) 308 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.210365+0000 mon.a (mon.0) 309 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.210400+0000 mon.a (mon.0) 310 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.210817+0000 mon.a (mon.0) 311 : cluster 0 mgrmap e38: y(active, since 2m), standbys: x 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.210988+0000 mon.a (mon.0) 312 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211009+0000 mon.a (mon.0) 313 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211022+0000 mon.a (mon.0) 314 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211032+0000 mon.a (mon.0) 315 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211046+0000 mon.a (mon.0) 316 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211067+0000 mon.a (mon.0) 317 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211081+0000 mon.a (mon.0) 318 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211106+0000 mon.a (mon.0) 319 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211117+0000 mon.a (mon.0) 320 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: cluster 2024-03-07T03:19:19.211129+0000 mon.a (mon.0) 321 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: audit 2024-03-07T03:19:19.195820+0000 mon.a (mon.0) 303 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.196173+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.196510+0000 mon.a (mon.0) 304 : cluster 1 mon.a calling monitor election 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.196630+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: audit 2024-03-07T03:19:19.198092+0000 mon.a (mon.0) 305 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:19:20.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: audit 2024-03-07T03:19:19.198258+0000 mon.a (mon.0) 306 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.200009+0000 mon.a (mon.0) 307 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.209781+0000 mon.a (mon.0) 308 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.210365+0000 mon.a (mon.0) 309 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.210400+0000 mon.a (mon.0) 310 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.210817+0000 mon.a (mon.0) 311 : cluster 0 mgrmap e38: y(active, since 2m), standbys: x 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.210988+0000 mon.a (mon.0) 312 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211009+0000 mon.a (mon.0) 313 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211022+0000 mon.a (mon.0) 314 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211032+0000 mon.a (mon.0) 315 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211046+0000 mon.a (mon.0) 316 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211067+0000 mon.a (mon.0) 317 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211081+0000 mon.a (mon.0) 318 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211106+0000 mon.a (mon.0) 319 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211117+0000 mon.a (mon.0) 320 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: cluster 2024-03-07T03:19:19.211129+0000 mon.a (mon.0) 321 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: audit 2024-03-07T03:19:19.231361+0000 mon.a (mon.0) 322 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: audit 2024-03-07T03:19:19.237005+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: audit 2024-03-07T03:19:19.247396+0000 mon.a (mon.0) 324 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[82222]: audit 2024-03-07T03:19:19.274715+0000 mon.a (mon.0) 325 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: audit 2024-03-07T03:19:19.231361+0000 mon.a (mon.0) 322 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: audit 2024-03-07T03:19:19.237005+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: audit 2024-03-07T03:19:19.247396+0000 mon.a (mon.0) 324 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.545 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:20 smithi148 bash[79819]: audit 2024-03-07T03:19:19.274715+0000 mon.a (mon.0) 325 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:20.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: audit 2024-03-07T03:19:19.195820+0000 mon.a (mon.0) 303 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.196173+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.196510+0000 mon.a (mon.0) 304 : cluster 1 mon.a calling monitor election 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.196630+0000 mon.c (mon.2) 3 : cluster 1 mon.c calling monitor election 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: audit 2024-03-07T03:19:19.198092+0000 mon.a (mon.0) 305 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: audit 2024-03-07T03:19:19.198258+0000 mon.a (mon.0) 306 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.200009+0000 mon.a (mon.0) 307 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.209781+0000 mon.a (mon.0) 308 : cluster 0 monmap e4: 3 mons at {a=[v2:172.21.15.148:3300/0,v1:172.21.15.148:6789/0],b=[v2:172.21.15.195:3300/0,v1:172.21.15.195:6789/0],c=[v2:172.21.15.148:3301/0,v1:172.21.15.148:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.210365+0000 mon.a (mon.0) 309 : cluster 0 fsmap foo:1 {0=foo.smithi195.gtlmjn=up:active} 1 up:standby 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.210400+0000 mon.a (mon.0) 310 : cluster 0 osdmap e387: 8 total, 8 up, 8 in 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.210817+0000 mon.a (mon.0) 311 : cluster 0 mgrmap e38: y(active, since 2m), standbys: x 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.210988+0000 mon.a (mon.0) 312 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:19:20.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211009+0000 mon.a (mon.0) 313 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211022+0000 mon.a (mon.0) 314 : cluster 4 osd.0's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211032+0000 mon.a (mon.0) 315 : cluster 4 osd.1's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211046+0000 mon.a (mon.0) 316 : cluster 4 osd.2's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211067+0000 mon.a (mon.0) 317 : cluster 4 osd.3's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211081+0000 mon.a (mon.0) 318 : cluster 4 osd.4's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211106+0000 mon.a (mon.0) 319 : cluster 4 osd.5's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211117+0000 mon.a (mon.0) 320 : cluster 4 osd.6's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: cluster 2024-03-07T03:19:19.211129+0000 mon.a (mon.0) 321 : cluster 4 osd.7's public address is not in '172.21.15.254/32,172.21.0.0/20,172.21.0.2/32,172.21.0.1/32' subnet 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: audit 2024-03-07T03:19:19.231361+0000 mon.a (mon.0) 322 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: audit 2024-03-07T03:19:19.237005+0000 mon.a (mon.0) 323 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: audit 2024-03-07T03:19:19.247396+0000 mon.a (mon.0) 324 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:20 smithi195 bash[66542]: audit 2024-03-07T03:19:19.274715+0000 mon.a (mon.0) 325 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:19:21.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:21 smithi148 bash[79819]: cluster 2024-03-07T03:19:20.203394+0000 mon.a (mon.0) 326 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2024-03-07T03:19:21.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:21 smithi148 bash[79819]: cluster 2024-03-07T03:19:20.240983+0000 mgr.y (mgr.34101) 91 : cluster 0 pgmap v169: 105 pgs: 105 active+clean; 41 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 6.5 MiB/s wr, 83 op/s 2024-03-07T03:19:21.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:21 smithi148 bash[82222]: cluster 2024-03-07T03:19:20.203394+0000 mon.a (mon.0) 326 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2024-03-07T03:19:21.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:21 smithi148 bash[82222]: cluster 2024-03-07T03:19:20.240983+0000 mgr.y (mgr.34101) 91 : cluster 0 pgmap v169: 105 pgs: 105 active+clean; 41 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 6.5 MiB/s wr, 83 op/s 2024-03-07T03:19:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:21 smithi195 bash[66542]: cluster 2024-03-07T03:19:20.203394+0000 mon.a (mon.0) 326 : cluster 0 osdmap e388: 8 total, 8 up, 8 in 2024-03-07T03:19:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:21 smithi195 bash[66542]: cluster 2024-03-07T03:19:20.240983+0000 mgr.y (mgr.34101) 91 : cluster 0 pgmap v169: 105 pgs: 105 active+clean; 41 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 7.9 MiB/s rd, 6.5 MiB/s wr, 83 op/s 2024-03-07T03:19:22.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:22 smithi148 bash[79819]: cluster 2024-03-07T03:19:21.217804+0000 mon.a (mon.0) 327 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:19:22.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:22 smithi148 bash[79819]: cluster 2024-03-07T03:19:22.040758+0000 mon.a (mon.0) 328 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2024-03-07T03:19:22.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:22 smithi148 bash[82222]: cluster 2024-03-07T03:19:21.217804+0000 mon.a (mon.0) 327 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:19:22.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:22 smithi148 bash[82222]: cluster 2024-03-07T03:19:22.040758+0000 mon.a (mon.0) 328 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2024-03-07T03:19:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:22 smithi195 bash[66542]: cluster 2024-03-07T03:19:21.217804+0000 mon.a (mon.0) 327 : cluster 0 osdmap e389: 8 total, 8 up, 8 in 2024-03-07T03:19:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:22 smithi195 bash[66542]: cluster 2024-03-07T03:19:22.040758+0000 mon.a (mon.0) 328 : cluster 0 osdmap e390: 8 total, 8 up, 8 in 2024-03-07T03:19:22.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:22 smithi195 bash[59134]: ts=2024-03-07T03:19:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:23.496 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Snaps (7815 ms) 2024-03-07T03:19:23.496 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-03-07T03:19:23.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:23 smithi195 bash[59134]: ts=2024-03-07T03:19:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:23.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:23 smithi195 bash[66542]: cluster 2024-03-07T03:19:22.241836+0000 mgr.y (mgr.34101) 92 : cluster 0 pgmap v172: 105 pgs: 105 active+clean; 41 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 6.7 MiB/s wr, 85 op/s 2024-03-07T03:19:23.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:23 smithi195 bash[66542]: audit 2024-03-07T03:19:22.316998+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:23.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:23 smithi195 bash[66542]: cluster 2024-03-07T03:19:23.044034+0000 mon.a (mon.0) 330 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2024-03-07T03:19:23.790 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:19:23 smithi148 bash[72584]: debug 2024-03-07T03:19:23.388+0000 7f6cc280d700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-07T03:19:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:23 smithi148 bash[79819]: cluster 2024-03-07T03:19:22.241836+0000 mgr.y (mgr.34101) 92 : cluster 0 pgmap v172: 105 pgs: 105 active+clean; 41 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 6.7 MiB/s wr, 85 op/s 2024-03-07T03:19:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:23 smithi148 bash[79819]: audit 2024-03-07T03:19:22.316998+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:23 smithi148 bash[79819]: cluster 2024-03-07T03:19:23.044034+0000 mon.a (mon.0) 330 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2024-03-07T03:19:23.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:23 smithi148 bash[82222]: cluster 2024-03-07T03:19:22.241836+0000 mgr.y (mgr.34101) 92 : cluster 0 pgmap v172: 105 pgs: 105 active+clean; 41 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 8.0 MiB/s rd, 6.7 MiB/s wr, 85 op/s 2024-03-07T03:19:23.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:23 smithi148 bash[82222]: audit 2024-03-07T03:19:22.316998+0000 mon.a (mon.0) 329 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:23.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:23 smithi148 bash[82222]: cluster 2024-03-07T03:19:23.044034+0000 mon.a (mon.0) 330 : cluster 0 osdmap e391: 8 total, 8 up, 8 in 2024-03-07T03:19:24.872 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:19:25.621 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.SnapDiscard (2125 ms) 2024-03-07T03:19:25.621 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-03-07T03:19:25.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[82222]: cluster 2024-03-07T03:19:24.243295+0000 mgr.y (mgr.34101) 93 : cluster 0 pgmap v174: 105 pgs: 105 active+clean; 65 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.0 MiB/s wr, 97 op/s 2024-03-07T03:19:25.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[82222]: cluster 2024-03-07T03:19:24.409162+0000 mon.a (mon.0) 331 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2024-03-07T03:19:25.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[82222]: audit 2024-03-07T03:19:25.318992+0000 mon.a (mon.0) 332 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:25.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[82222]: audit 2024-03-07T03:19:25.328586+0000 mon.a (mon.0) 333 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:25.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[79819]: cluster 2024-03-07T03:19:24.243295+0000 mgr.y (mgr.34101) 93 : cluster 0 pgmap v174: 105 pgs: 105 active+clean; 65 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.0 MiB/s wr, 97 op/s 2024-03-07T03:19:25.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[79819]: cluster 2024-03-07T03:19:24.409162+0000 mon.a (mon.0) 331 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2024-03-07T03:19:25.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[79819]: audit 2024-03-07T03:19:25.318992+0000 mon.a (mon.0) 332 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:25.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:25 smithi148 bash[79819]: audit 2024-03-07T03:19:25.328586+0000 mon.a (mon.0) 333 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:25 smithi195 bash[66542]: cluster 2024-03-07T03:19:24.243295+0000 mgr.y (mgr.34101) 93 : cluster 0 pgmap v174: 105 pgs: 105 active+clean; 65 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 7.0 MiB/s rd, 6.0 MiB/s wr, 97 op/s 2024-03-07T03:19:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:25 smithi195 bash[66542]: cluster 2024-03-07T03:19:24.409162+0000 mon.a (mon.0) 331 : cluster 0 osdmap e392: 8 total, 8 up, 8 in 2024-03-07T03:19:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:25 smithi195 bash[66542]: audit 2024-03-07T03:19:25.318992+0000 mon.a (mon.0) 332 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:25 smithi195 bash[66542]: audit 2024-03-07T03:19:25.328586+0000 mon.a (mon.0) 333 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:26.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:26 smithi148 bash[79819]: cluster 2024-03-07T03:19:25.418104+0000 mon.a (mon.0) 334 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2024-03-07T03:19:26.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:26 smithi148 bash[79819]: audit 2024-03-07T03:19:26.372660+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:26.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:26 smithi148 bash[79819]: audit 2024-03-07T03:19:26.383894+0000 mon.a (mon.0) 336 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:26.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:26 smithi148 bash[82222]: cluster 2024-03-07T03:19:25.418104+0000 mon.a (mon.0) 334 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2024-03-07T03:19:26.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:26 smithi148 bash[82222]: audit 2024-03-07T03:19:26.372660+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:26.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:26 smithi148 bash[82222]: audit 2024-03-07T03:19:26.383894+0000 mon.a (mon.0) 336 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:26.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:26 smithi195 bash[66542]: cluster 2024-03-07T03:19:25.418104+0000 mon.a (mon.0) 334 : cluster 0 osdmap e393: 8 total, 8 up, 8 in 2024-03-07T03:19:26.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:26 smithi195 bash[66542]: audit 2024-03-07T03:19:26.372660+0000 mon.a (mon.0) 335 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:26.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:26 smithi195 bash[66542]: audit 2024-03-07T03:19:26.383894+0000 mon.a (mon.0) 336 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:26.838 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.CloneDiscard (1217 ms) 2024-03-07T03:19:26.838 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-03-07T03:19:27.426 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:19:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:19:27] "GET /metrics HTTP/1.1" 200 36326 "" "Prometheus/2.43.0" 2024-03-07T03:19:27.760 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:27 smithi148 bash[79819]: cluster 2024-03-07T03:19:26.244019+0000 mgr.y (mgr.34101) 94 : cluster 0 pgmap v177: 105 pgs: 105 active+clean; 65 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.7 MiB/s wr, 93 op/s 2024-03-07T03:19:27.761 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:27 smithi148 bash[79819]: cluster 2024-03-07T03:19:26.437740+0000 mon.a (mon.0) 337 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2024-03-07T03:19:27.761 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:27 smithi148 bash[79819]: audit 2024-03-07T03:19:27.300550+0000 mon.a (mon.0) 338 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:27.761 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:27 smithi148 bash[82222]: cluster 2024-03-07T03:19:26.244019+0000 mgr.y (mgr.34101) 94 : cluster 0 pgmap v177: 105 pgs: 105 active+clean; 65 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.7 MiB/s wr, 93 op/s 2024-03-07T03:19:27.761 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:27 smithi148 bash[82222]: cluster 2024-03-07T03:19:26.437740+0000 mon.a (mon.0) 337 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2024-03-07T03:19:27.761 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:27 smithi148 bash[82222]: audit 2024-03-07T03:19:27.300550+0000 mon.a (mon.0) 338 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:27.811 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.CloneShrink (973 ms) 2024-03-07T03:19:27.811 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-03-07T03:19:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:27 smithi195 bash[66542]: cluster 2024-03-07T03:19:26.244019+0000 mgr.y (mgr.34101) 94 : cluster 0 pgmap v177: 105 pgs: 105 active+clean; 65 MiB data, 529 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.7 MiB/s wr, 93 op/s 2024-03-07T03:19:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:27 smithi195 bash[66542]: cluster 2024-03-07T03:19:26.437740+0000 mon.a (mon.0) 337 : cluster 0 osdmap e394: 8 total, 8 up, 8 in 2024-03-07T03:19:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:27 smithi195 bash[66542]: audit 2024-03-07T03:19:27.300550+0000 mon.a (mon.0) 338 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:28.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:28 smithi148 bash[79819]: cluster 2024-03-07T03:19:27.433734+0000 mon.a (mon.0) 339 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2024-03-07T03:19:28.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:28 smithi148 bash[82222]: cluster 2024-03-07T03:19:27.433734+0000 mon.a (mon.0) 339 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2024-03-07T03:19:28.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:28 smithi195 bash[66542]: cluster 2024-03-07T03:19:27.433734+0000 mon.a (mon.0) 339 : cluster 0 osdmap e395: 8 total, 8 up, 8 in 2024-03-07T03:19:28.959 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.CloneExpand (1148 ms) 2024-03-07T03:19:28.959 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-03-07T03:19:29.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:29 smithi148 bash[79819]: cluster 2024-03-07T03:19:28.245280+0000 mgr.y (mgr.34101) 95 : cluster 0 pgmap v180: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 27 KiB/s wr, 388 op/s 2024-03-07T03:19:29.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:29 smithi148 bash[79819]: cluster 2024-03-07T03:19:28.458796+0000 mon.a (mon.0) 340 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2024-03-07T03:19:29.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:29 smithi148 bash[82222]: cluster 2024-03-07T03:19:28.245280+0000 mgr.y (mgr.34101) 95 : cluster 0 pgmap v180: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 27 KiB/s wr, 388 op/s 2024-03-07T03:19:29.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:29 smithi148 bash[82222]: cluster 2024-03-07T03:19:28.458796+0000 mon.a (mon.0) 340 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2024-03-07T03:19:29.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:29 smithi195 bash[66542]: cluster 2024-03-07T03:19:28.245280+0000 mgr.y (mgr.34101) 95 : cluster 0 pgmap v180: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 27 KiB/s wr, 388 op/s 2024-03-07T03:19:29.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:29 smithi195 bash[66542]: cluster 2024-03-07T03:19:28.458796+0000 mon.a (mon.0) 340 : cluster 0 osdmap e396: 8 total, 8 up, 8 in 2024-03-07T03:19:30.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:30 smithi148 bash[79819]: cluster 2024-03-07T03:19:29.464723+0000 mon.a (mon.0) 341 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2024-03-07T03:19:30.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:30 smithi148 bash[82222]: cluster 2024-03-07T03:19:29.464723+0000 mon.a (mon.0) 341 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2024-03-07T03:19:30.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:30 smithi195 bash[66542]: cluster 2024-03-07T03:19:29.464723+0000 mon.a (mon.0) 341 : cluster 0 osdmap e397: 8 total, 8 up, 8 in 2024-03-07T03:19:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:31 smithi148 bash[79819]: cluster 2024-03-07T03:19:30.245957+0000 mgr.y (mgr.34101) 96 : cluster 0 pgmap v183: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 27 KiB/s wr, 388 op/s 2024-03-07T03:19:31.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:31 smithi148 bash[79819]: cluster 2024-03-07T03:19:30.475013+0000 mon.a (mon.0) 342 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2024-03-07T03:19:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:31 smithi148 bash[82222]: cluster 2024-03-07T03:19:30.245957+0000 mgr.y (mgr.34101) 96 : cluster 0 pgmap v183: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 27 KiB/s wr, 388 op/s 2024-03-07T03:19:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:31 smithi148 bash[82222]: cluster 2024-03-07T03:19:30.475013+0000 mon.a (mon.0) 342 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2024-03-07T03:19:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:31 smithi195 bash[66542]: cluster 2024-03-07T03:19:30.245957+0000 mgr.y (mgr.34101) 96 : cluster 0 pgmap v183: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 17 MiB/s rd, 27 KiB/s wr, 388 op/s 2024-03-07T03:19:31.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:31 smithi195 bash[66542]: cluster 2024-03-07T03:19:30.475013+0000 mon.a (mon.0) 342 : cluster 0 osdmap e398: 8 total, 8 up, 8 in 2024-03-07T03:19:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:32 smithi195 bash[66542]: cluster 2024-03-07T03:19:31.484244+0000 mon.a (mon.0) 343 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2024-03-07T03:19:32.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:32 smithi195 bash[59134]: ts=2024-03-07T03:19:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:32.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:32 smithi148 bash[79819]: cluster 2024-03-07T03:19:31.484244+0000 mon.a (mon.0) 343 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2024-03-07T03:19:32.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:32 smithi148 bash[82222]: cluster 2024-03-07T03:19:31.484244+0000 mon.a (mon.0) 343 : cluster 0 osdmap e399: 8 total, 8 up, 8 in 2024-03-07T03:19:33.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:33 smithi195 bash[66542]: cluster 2024-03-07T03:19:32.246633+0000 mgr.y (mgr.34101) 97 : cluster 0 pgmap v186: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:33.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:33 smithi195 bash[66542]: cluster 2024-03-07T03:19:32.500962+0000 mon.a (mon.0) 344 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2024-03-07T03:19:33.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:33 smithi195 bash[59134]: ts=2024-03-07T03:19:33.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:33.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:33 smithi148 bash[82222]: cluster 2024-03-07T03:19:32.246633+0000 mgr.y (mgr.34101) 97 : cluster 0 pgmap v186: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:33.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:33 smithi148 bash[82222]: cluster 2024-03-07T03:19:32.500962+0000 mon.a (mon.0) 344 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2024-03-07T03:19:33.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:33 smithi148 bash[79819]: cluster 2024-03-07T03:19:32.246633+0000 mgr.y (mgr.34101) 97 : cluster 0 pgmap v186: 105 pgs: 105 active+clean; 65 MiB data, 609 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:19:33.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:33 smithi148 bash[79819]: cluster 2024-03-07T03:19:32.500962+0000 mon.a (mon.0) 344 : cluster 0 osdmap e400: 8 total, 8 up, 8 in 2024-03-07T03:19:34.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:34 smithi148 bash[79819]: cluster 2024-03-07T03:19:33.506204+0000 mon.a (mon.0) 345 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2024-03-07T03:19:34.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:34 smithi148 bash[82222]: cluster 2024-03-07T03:19:33.506204+0000 mon.a (mon.0) 345 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2024-03-07T03:19:34.796 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:34 smithi195 bash[66542]: cluster 2024-03-07T03:19:33.506204+0000 mon.a (mon.0) 345 : cluster 0 osdmap e401: 8 total, 8 up, 8 in 2024-03-07T03:19:35.730 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.CloneHideParent (6771 ms) 2024-03-07T03:19:35.730 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Clone 2024-03-07T03:19:35.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:35 smithi148 bash[79819]: cluster 2024-03-07T03:19:34.247306+0000 mgr.y (mgr.34101) 98 : cluster 0 pgmap v189: 105 pgs: 105 active+clean; 65 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:19:35.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:35 smithi148 bash[79819]: cluster 2024-03-07T03:19:34.517528+0000 mon.a (mon.0) 346 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2024-03-07T03:19:35.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:35 smithi148 bash[82222]: cluster 2024-03-07T03:19:34.247306+0000 mgr.y (mgr.34101) 98 : cluster 0 pgmap v189: 105 pgs: 105 active+clean; 65 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:19:35.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:35 smithi148 bash[82222]: cluster 2024-03-07T03:19:34.517528+0000 mon.a (mon.0) 346 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2024-03-07T03:19:35.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:35 smithi195 bash[66542]: cluster 2024-03-07T03:19:34.247306+0000 mgr.y (mgr.34101) 98 : cluster 0 pgmap v189: 105 pgs: 105 active+clean; 65 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:19:35.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:35 smithi195 bash[66542]: cluster 2024-03-07T03:19:34.517528+0000 mon.a (mon.0) 346 : cluster 0 osdmap e402: 8 total, 8 up, 8 in 2024-03-07T03:19:36.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:36 smithi148 bash[79819]: cluster 2024-03-07T03:19:35.517234+0000 mon.a (mon.0) 347 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2024-03-07T03:19:36.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:36 smithi148 bash[82222]: cluster 2024-03-07T03:19:35.517234+0000 mon.a (mon.0) 347 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2024-03-07T03:19:36.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:36 smithi195 bash[66542]: cluster 2024-03-07T03:19:35.517234+0000 mon.a (mon.0) 347 : cluster 0 osdmap e403: 8 total, 8 up, 8 in 2024-03-07T03:19:37.520 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:19:37] "GET /metrics HTTP/1.1" 200 36402 "" "Prometheus/2.43.0" 2024-03-07T03:19:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: cluster 2024-03-07T03:19:36.247974+0000 mgr.y (mgr.34101) 99 : cluster 0 pgmap v192: 105 pgs: 105 active+clean; 65 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:19:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: cluster 2024-03-07T03:19:36.543874+0000 mon.a (mon.0) 348 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2024-03-07T03:19:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.744326+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.764051+0000 mon.a (mon.0) 350 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.766754+0000 mon.a (mon.0) 351 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.768767+0000 mon.a (mon.0) 352 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.780644+0000 mon.a (mon.0) 353 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.856172+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.857945+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[79819]: audit 2024-03-07T03:19:36.859373+0000 mon.a (mon.0) 356 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: cluster 2024-03-07T03:19:36.247974+0000 mgr.y (mgr.34101) 99 : cluster 0 pgmap v192: 105 pgs: 105 active+clean; 65 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: cluster 2024-03-07T03:19:36.543874+0000 mon.a (mon.0) 348 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.744326+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.764051+0000 mon.a (mon.0) 350 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.766754+0000 mon.a (mon.0) 351 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:37.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.768767+0000 mon.a (mon.0) 352 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:37.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.780644+0000 mon.a (mon.0) 353 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.856172+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:37.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.857945+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:37.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:37 smithi148 bash[82222]: audit 2024-03-07T03:19:36.859373+0000 mon.a (mon.0) 356 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: cluster 2024-03-07T03:19:36.247974+0000 mgr.y (mgr.34101) 99 : cluster 0 pgmap v192: 105 pgs: 105 active+clean; 65 MiB data, 614 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 18 KiB/s wr, 327 op/s 2024-03-07T03:19:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: cluster 2024-03-07T03:19:36.543874+0000 mon.a (mon.0) 348 : cluster 0 osdmap e404: 8 total, 8 up, 8 in 2024-03-07T03:19:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.744326+0000 mon.a (mon.0) 349 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.764051+0000 mon.a (mon.0) 350 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.766754+0000 mon.a (mon.0) 351 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.768767+0000 mon.a (mon.0) 352 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:19:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.780644+0000 mon.a (mon.0) 353 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.856172+0000 mon.a (mon.0) 354 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.857945+0000 mon.a (mon.0) 355 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:37 smithi195 bash[66542]: audit 2024-03-07T03:19:36.859373+0000 mon.a (mon.0) 356 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:38.394 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Clone (2664 ms) 2024-03-07T03:19:38.395 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-03-07T03:19:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:38 smithi148 bash[79819]: cephadm 2024-03-07T03:19:36.855404+0000 mgr.y (mgr.34101) 100 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2024-03-07T03:19:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:38 smithi148 bash[79819]: cephadm 2024-03-07T03:19:36.861038+0000 mgr.y (mgr.34101) 101 : cephadm 1 Reconfiguring daemon mon.a on smithi148 2024-03-07T03:19:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:38 smithi148 bash[79819]: cluster 2024-03-07T03:19:37.767341+0000 mon.a (mon.0) 357 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2024-03-07T03:19:38.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:38 smithi148 bash[82222]: cephadm 2024-03-07T03:19:36.855404+0000 mgr.y (mgr.34101) 100 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2024-03-07T03:19:38.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:38 smithi148 bash[82222]: cephadm 2024-03-07T03:19:36.861038+0000 mgr.y (mgr.34101) 101 : cephadm 1 Reconfiguring daemon mon.a on smithi148 2024-03-07T03:19:38.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:38 smithi148 bash[82222]: cluster 2024-03-07T03:19:37.767341+0000 mon.a (mon.0) 357 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2024-03-07T03:19:38.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:38 smithi195 bash[66542]: cephadm 2024-03-07T03:19:36.855404+0000 mgr.y (mgr.34101) 100 : cephadm 1 Reconfiguring mon.a (monmap changed)... 2024-03-07T03:19:38.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:38 smithi195 bash[66542]: cephadm 2024-03-07T03:19:36.861038+0000 mgr.y (mgr.34101) 101 : cephadm 1 Reconfiguring daemon mon.a on smithi148 2024-03-07T03:19:38.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:38 smithi195 bash[66542]: cluster 2024-03-07T03:19:37.767341+0000 mon.a (mon.0) 357 : cluster 0 osdmap e405: 8 total, 8 up, 8 in 2024-03-07T03:19:39.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[79819]: cluster 2024-03-07T03:19:38.249093+0000 mgr.y (mgr.34101) 102 : cluster 0 pgmap v195: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.0 MiB/s wr, 287 op/s 2024-03-07T03:19:39.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[79819]: audit 2024-03-07T03:19:39.367683+0000 mon.a (mon.0) 358 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:39.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[79819]: audit 2024-03-07T03:19:39.375801+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:39.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[79819]: audit 2024-03-07T03:19:39.379781+0000 mon.a (mon.0) 360 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:19:39.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[79819]: audit 2024-03-07T03:19:39.381285+0000 mon.a (mon.0) 361 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:19:39.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[79819]: audit 2024-03-07T03:19:39.382670+0000 mon.a (mon.0) 362 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:39.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[82222]: cluster 2024-03-07T03:19:38.249093+0000 mgr.y (mgr.34101) 102 : cluster 0 pgmap v195: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.0 MiB/s wr, 287 op/s 2024-03-07T03:19:39.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[82222]: audit 2024-03-07T03:19:39.367683+0000 mon.a (mon.0) 358 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:39.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[82222]: audit 2024-03-07T03:19:39.375801+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:39.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[82222]: audit 2024-03-07T03:19:39.379781+0000 mon.a (mon.0) 360 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:19:39.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[82222]: audit 2024-03-07T03:19:39.381285+0000 mon.a (mon.0) 361 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:19:39.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:39 smithi148 bash[82222]: audit 2024-03-07T03:19:39.382670+0000 mon.a (mon.0) 362 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:39.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:39 smithi195 bash[66542]: cluster 2024-03-07T03:19:38.249093+0000 mgr.y (mgr.34101) 102 : cluster 0 pgmap v195: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.0 MiB/s wr, 287 op/s 2024-03-07T03:19:39.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:39 smithi195 bash[66542]: audit 2024-03-07T03:19:39.367683+0000 mon.a (mon.0) 358 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:39.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:39 smithi195 bash[66542]: audit 2024-03-07T03:19:39.375801+0000 mon.a (mon.0) 359 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:39.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:39 smithi195 bash[66542]: audit 2024-03-07T03:19:39.379781+0000 mon.a (mon.0) 360 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:19:39.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:39 smithi195 bash[66542]: audit 2024-03-07T03:19:39.381285+0000 mon.a (mon.0) 361 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:19:39.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:39 smithi195 bash[66542]: audit 2024-03-07T03:19:39.382670+0000 mon.a (mon.0) 362 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:40.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:40 smithi195 bash[66542]: cephadm 2024-03-07T03:19:39.379228+0000 mgr.y (mgr.34101) 103 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2024-03-07T03:19:40.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:40 smithi195 bash[66542]: cephadm 2024-03-07T03:19:39.384350+0000 mgr.y (mgr.34101) 104 : cephadm 1 Reconfiguring daemon mgr.y on smithi148 2024-03-07T03:19:40.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:40 smithi195 bash[66542]: cluster 2024-03-07T03:19:39.546351+0000 mon.a (mon.0) 363 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2024-03-07T03:19:40.880 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:40 smithi148 bash[79819]: cephadm 2024-03-07T03:19:39.379228+0000 mgr.y (mgr.34101) 103 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2024-03-07T03:19:40.880 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:40 smithi148 bash[79819]: cephadm 2024-03-07T03:19:39.384350+0000 mgr.y (mgr.34101) 104 : cephadm 1 Reconfiguring daemon mgr.y on smithi148 2024-03-07T03:19:40.880 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:40 smithi148 bash[79819]: cluster 2024-03-07T03:19:39.546351+0000 mon.a (mon.0) 363 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2024-03-07T03:19:40.883 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:40 smithi148 bash[82222]: cephadm 2024-03-07T03:19:39.379228+0000 mgr.y (mgr.34101) 103 : cephadm 1 Reconfiguring mgr.y (monmap changed)... 2024-03-07T03:19:40.883 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:40 smithi148 bash[82222]: cephadm 2024-03-07T03:19:39.384350+0000 mgr.y (mgr.34101) 104 : cephadm 1 Reconfiguring daemon mgr.y on smithi148 2024-03-07T03:19:40.883 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:40 smithi148 bash[82222]: cluster 2024-03-07T03:19:39.546351+0000 mon.a (mon.0) 363 : cluster 0 osdmap e406: 8 total, 8 up, 8 in 2024-03-07T03:19:41.217 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.CloneFlatten (2823 ms) 2024-03-07T03:19:41.217 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Stress 2024-03-07T03:19:41.275 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 78730065~5156015 2024-03-07T03:19:41.305 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 8172181~1355522 2024-03-07T03:19:41.307 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 68088590~6915246 2024-03-07T03:19:41.335 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 50943595~7755929 2024-03-07T03:19:41.335 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 41128731~2945176 2024-03-07T03:19:41.353 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 38210266~6744667 2024-03-07T03:19:41.356 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 18990254~3060518 2024-03-07T03:19:41.375 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 4127484~3647209 2024-03-07T03:19:41.375 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap0 2024-03-07T03:19:41.579 INFO:tasks.workunit.client.0.smithi148.stdout:clone image238 -> image239 2024-03-07T03:19:41.671 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 2506078~2279583 2024-03-07T03:19:41.691 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 23620088~7839832 2024-03-07T03:19:41.708 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 6852247~7584056 2024-03-07T03:19:41.734 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 51220354~4697494 2024-03-07T03:19:41.742 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 51574088~1303214 2024-03-07T03:19:41.749 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 79963027~2204181 2024-03-07T03:19:41.777 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 57691844~2867027 2024-03-07T03:19:41.794 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 46165590~3854582 2024-03-07T03:19:41.800 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:19:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:41 smithi195 bash[66542]: cluster 2024-03-07T03:19:40.249750+0000 mgr.y (mgr.34101) 105 : cluster 0 pgmap v197: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 7.6 MiB/s wr, 242 op/s 2024-03-07T03:19:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:41 smithi195 bash[66542]: cluster 2024-03-07T03:19:40.566248+0000 mon.a (mon.0) 364 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2024-03-07T03:19:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:41 smithi195 bash[66542]: audit 2024-03-07T03:19:41.460460+0000 mon.a (mon.0) 365 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:41 smithi195 bash[66542]: audit 2024-03-07T03:19:41.472672+0000 mon.a (mon.0) 366 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:41 smithi195 bash[66542]: audit 2024-03-07T03:19:41.502870+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:41 smithi195 bash[66542]: audit 2024-03-07T03:19:41.504172+0000 mon.a (mon.0) 368 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:41 smithi195 bash[66542]: audit 2024-03-07T03:19:41.505578+0000 mon.a (mon.0) 369 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:41.973 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[79819]: cluster 2024-03-07T03:19:40.249750+0000 mgr.y (mgr.34101) 105 : cluster 0 pgmap v197: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 7.6 MiB/s wr, 242 op/s 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[79819]: cluster 2024-03-07T03:19:40.566248+0000 mon.a (mon.0) 364 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[79819]: audit 2024-03-07T03:19:41.460460+0000 mon.a (mon.0) 365 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[79819]: audit 2024-03-07T03:19:41.472672+0000 mon.a (mon.0) 366 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[79819]: audit 2024-03-07T03:19:41.502870+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[79819]: audit 2024-03-07T03:19:41.504172+0000 mon.a (mon.0) 368 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[79819]: audit 2024-03-07T03:19:41.505578+0000 mon.a (mon.0) 369 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[82222]: cluster 2024-03-07T03:19:40.249750+0000 mgr.y (mgr.34101) 105 : cluster 0 pgmap v197: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 7.6 MiB/s wr, 242 op/s 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[82222]: cluster 2024-03-07T03:19:40.566248+0000 mon.a (mon.0) 364 : cluster 0 osdmap e407: 8 total, 8 up, 8 in 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[82222]: audit 2024-03-07T03:19:41.460460+0000 mon.a (mon.0) 365 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[82222]: audit 2024-03-07T03:19:41.472672+0000 mon.a (mon.0) 366 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:41.974 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[82222]: audit 2024-03-07T03:19:41.502870+0000 mon.a (mon.0) 367 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:41.975 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[82222]: audit 2024-03-07T03:19:41.504172+0000 mon.a (mon.0) 368 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:41.975 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:41 smithi148 bash[82222]: audit 2024-03-07T03:19:41.505578+0000 mon.a (mon.0) 369 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:42.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:42 smithi195 bash[59134]: ts=2024-03-07T03:19:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:42.610 INFO:tasks.workunit.client.0.smithi148.stdout:clone image239 -> image240 2024-03-07T03:19:42.747 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 40065523~5202276 2024-03-07T03:19:42.788 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 16536061~2993168 2024-03-07T03:19:42.810 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 60069483~5696783 2024-03-07T03:19:42.826 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[79819]: cephadm 2024-03-07T03:19:41.502326+0000 mgr.y (mgr.34101) 106 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:19:42.826 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[79819]: cephadm 2024-03-07T03:19:41.507282+0000 mgr.y (mgr.34101) 107 : cephadm 1 Reconfiguring daemon mon.c on smithi148 2024-03-07T03:19:42.826 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[79819]: cluster 2024-03-07T03:19:41.565824+0000 mon.a (mon.0) 370 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2024-03-07T03:19:42.826 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[79819]: audit 2024-03-07T03:19:42.301128+0000 mon.a (mon.0) 371 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:42.826 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[82222]: cephadm 2024-03-07T03:19:41.502326+0000 mgr.y (mgr.34101) 106 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:19:42.826 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[82222]: cephadm 2024-03-07T03:19:41.507282+0000 mgr.y (mgr.34101) 107 : cephadm 1 Reconfiguring daemon mon.c on smithi148 2024-03-07T03:19:42.826 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[82222]: cluster 2024-03-07T03:19:41.565824+0000 mon.a (mon.0) 370 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2024-03-07T03:19:42.827 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:42 smithi148 bash[82222]: audit 2024-03-07T03:19:42.301128+0000 mon.a (mon.0) 371 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:42.837 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 17145909~4414679 2024-03-07T03:19:42.846 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 16450497~3562265 2024-03-07T03:19:42.860 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 63095387~4496177 2024-03-07T03:19:42.880 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 23552976~3569749 2024-03-07T03:19:42.898 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 61741229~6881210 2024-03-07T03:19:42.900 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:19:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:42 smithi195 bash[66542]: cephadm 2024-03-07T03:19:41.502326+0000 mgr.y (mgr.34101) 106 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-03-07T03:19:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:42 smithi195 bash[66542]: cephadm 2024-03-07T03:19:41.507282+0000 mgr.y (mgr.34101) 107 : cephadm 1 Reconfiguring daemon mon.c on smithi148 2024-03-07T03:19:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:42 smithi195 bash[66542]: cluster 2024-03-07T03:19:41.565824+0000 mon.a (mon.0) 370 : cluster 0 osdmap e408: 8 total, 8 up, 8 in 2024-03-07T03:19:43.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:42 smithi195 bash[66542]: audit 2024-03-07T03:19:42.301128+0000 mon.a (mon.0) 371 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:43.609 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 43481951~1484612 2024-03-07T03:19:43.614 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 61508082~6965364 2024-03-07T03:19:43.615 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 17558154~2781351 2024-03-07T03:19:43.615 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 14036987~7253382 2024-03-07T03:19:43.650 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 36273788~8141134 2024-03-07T03:19:43.650 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 57227944~1482321 2024-03-07T03:19:43.652 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 73793461~5100577 2024-03-07T03:19:43.652 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 41648177~5018780 2024-03-07T03:19:43.714 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap3 2024-03-07T03:19:43.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:43 smithi195 bash[66542]: cluster 2024-03-07T03:19:42.250410+0000 mgr.y (mgr.34101) 108 : cluster 0 pgmap v200: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 8.0 MiB/s wr, 256 op/s 2024-03-07T03:19:43.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:43 smithi195 bash[66542]: cluster 2024-03-07T03:19:42.588624+0000 mon.a (mon.0) 372 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2024-03-07T03:19:43.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:43 smithi195 bash[59134]: ts=2024-03-07T03:19:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:44.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:43 smithi148 bash[79819]: cluster 2024-03-07T03:19:42.250410+0000 mgr.y (mgr.34101) 108 : cluster 0 pgmap v200: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 8.0 MiB/s wr, 256 op/s 2024-03-07T03:19:44.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:43 smithi148 bash[79819]: cluster 2024-03-07T03:19:42.588624+0000 mon.a (mon.0) 372 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2024-03-07T03:19:44.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:43 smithi148 bash[82222]: cluster 2024-03-07T03:19:42.250410+0000 mgr.y (mgr.34101) 108 : cluster 0 pgmap v200: 105 pgs: 105 active+clean; 101 MiB data, 687 MiB used, 715 GiB / 715 GiB avail; 3.7 MiB/s rd, 8.0 MiB/s wr, 256 op/s 2024-03-07T03:19:44.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:43 smithi148 bash[82222]: cluster 2024-03-07T03:19:42.588624+0000 mon.a (mon.0) 372 : cluster 0 osdmap e409: 8 total, 8 up, 8 in 2024-03-07T03:19:44.665 INFO:tasks.workunit.client.0.smithi148.stdout:clone image240 -> image241 2024-03-07T03:19:44.825 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 135272092 2024-03-07T03:19:44.906 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: cluster 2024-03-07T03:19:43.595465+0000 mon.a (mon.0) 373 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2024-03-07T03:19:44.906 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: audit 2024-03-07T03:19:43.626896+0000 mon.a (mon.0) 374 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: audit 2024-03-07T03:19:43.640240+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: cephadm 2024-03-07T03:19:43.680664+0000 mgr.y (mgr.34101) 109 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: audit 2024-03-07T03:19:43.681657+0000 mon.a (mon.0) 376 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: audit 2024-03-07T03:19:43.689313+0000 mon.a (mon.0) 377 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: cephadm 2024-03-07T03:19:43.694210+0000 mgr.y (mgr.34101) 110 : cephadm 1 Reconfiguring daemon osd.0 on smithi148 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[79819]: cluster 2024-03-07T03:19:44.251134+0000 mgr.y (mgr.34101) 111 : cluster 0 pgmap v203: 105 pgs: 105 active+clean; 285 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 42 MiB/s wr, 511 op/s 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: cluster 2024-03-07T03:19:43.595465+0000 mon.a (mon.0) 373 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: audit 2024-03-07T03:19:43.626896+0000 mon.a (mon.0) 374 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: audit 2024-03-07T03:19:43.640240+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: cephadm 2024-03-07T03:19:43.680664+0000 mgr.y (mgr.34101) 109 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2024-03-07T03:19:44.907 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: audit 2024-03-07T03:19:43.681657+0000 mon.a (mon.0) 376 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:19:44.908 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: audit 2024-03-07T03:19:43.689313+0000 mon.a (mon.0) 377 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:44.908 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: cephadm 2024-03-07T03:19:43.694210+0000 mgr.y (mgr.34101) 110 : cephadm 1 Reconfiguring daemon osd.0 on smithi148 2024-03-07T03:19:44.908 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:44 smithi148 bash[82222]: cluster 2024-03-07T03:19:44.251134+0000 mgr.y (mgr.34101) 111 : cluster 0 pgmap v203: 105 pgs: 105 active+clean; 285 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 42 MiB/s wr, 511 op/s 2024-03-07T03:19:45.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: cluster 2024-03-07T03:19:43.595465+0000 mon.a (mon.0) 373 : cluster 0 osdmap e410: 8 total, 8 up, 8 in 2024-03-07T03:19:45.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: audit 2024-03-07T03:19:43.626896+0000 mon.a (mon.0) 374 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:45.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: audit 2024-03-07T03:19:43.640240+0000 mon.a (mon.0) 375 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:45.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: cephadm 2024-03-07T03:19:43.680664+0000 mgr.y (mgr.34101) 109 : cephadm 1 Reconfiguring osd.0 (monmap changed)... 2024-03-07T03:19:45.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: audit 2024-03-07T03:19:43.681657+0000 mon.a (mon.0) 376 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:19:45.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: audit 2024-03-07T03:19:43.689313+0000 mon.a (mon.0) 377 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:45.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: cephadm 2024-03-07T03:19:43.694210+0000 mgr.y (mgr.34101) 110 : cephadm 1 Reconfiguring daemon osd.0 on smithi148 2024-03-07T03:19:45.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:44 smithi195 bash[66542]: cluster 2024-03-07T03:19:44.251134+0000 mgr.y (mgr.34101) 111 : cluster 0 pgmap v203: 105 pgs: 105 active+clean; 285 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 42 MiB/s wr, 511 op/s 2024-03-07T03:19:45.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Stress (3985 ms) 2024-03-07T03:19:45.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-03-07T03:19:45.626 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (424 ms) 2024-03-07T03:19:45.626 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-03-07T03:19:46.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:45 smithi148 bash[82222]: cluster 2024-03-07T03:19:44.647544+0000 mon.a (mon.0) 378 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2024-03-07T03:19:46.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:45 smithi148 bash[79819]: cluster 2024-03-07T03:19:44.647544+0000 mon.a (mon.0) 378 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2024-03-07T03:19:46.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:45 smithi195 bash[66542]: cluster 2024-03-07T03:19:44.647544+0000 mon.a (mon.0) 378 : cluster 0 osdmap e411: 8 total, 8 up, 8 in 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: audit 2024-03-07T03:19:45.906959+0000 mon.a (mon.0) 379 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: audit 2024-03-07T03:19:45.913521+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: cephadm 2024-03-07T03:19:45.920401+0000 mgr.y (mgr.34101) 112 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: audit 2024-03-07T03:19:45.920943+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: audit 2024-03-07T03:19:45.922267+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: cephadm 2024-03-07T03:19:45.926019+0000 mgr.y (mgr.34101) 113 : cephadm 1 Reconfiguring daemon osd.1 on smithi148 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: cluster 2024-03-07T03:19:46.251808+0000 mgr.y (mgr.34101) 114 : cluster 0 pgmap v205: 105 pgs: 105 active+clean; 285 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 436 op/s 2024-03-07T03:19:47.156 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[79819]: cluster 2024-03-07T03:19:46.654929+0000 mon.a (mon.0) 383 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: audit 2024-03-07T03:19:45.906959+0000 mon.a (mon.0) 379 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: audit 2024-03-07T03:19:45.913521+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: cephadm 2024-03-07T03:19:45.920401+0000 mgr.y (mgr.34101) 112 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: audit 2024-03-07T03:19:45.920943+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: audit 2024-03-07T03:19:45.922267+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: cephadm 2024-03-07T03:19:45.926019+0000 mgr.y (mgr.34101) 113 : cephadm 1 Reconfiguring daemon osd.1 on smithi148 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: cluster 2024-03-07T03:19:46.251808+0000 mgr.y (mgr.34101) 114 : cluster 0 pgmap v205: 105 pgs: 105 active+clean; 285 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 436 op/s 2024-03-07T03:19:47.157 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:46 smithi148 bash[82222]: cluster 2024-03-07T03:19:46.654929+0000 mon.a (mon.0) 383 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2024-03-07T03:19:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: audit 2024-03-07T03:19:45.906959+0000 mon.a (mon.0) 379 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: audit 2024-03-07T03:19:45.913521+0000 mon.a (mon.0) 380 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: cephadm 2024-03-07T03:19:45.920401+0000 mgr.y (mgr.34101) 112 : cephadm 1 Reconfiguring osd.1 (monmap changed)... 2024-03-07T03:19:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: audit 2024-03-07T03:19:45.920943+0000 mon.a (mon.0) 381 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:19:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: audit 2024-03-07T03:19:45.922267+0000 mon.a (mon.0) 382 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:47.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: cephadm 2024-03-07T03:19:45.926019+0000 mgr.y (mgr.34101) 113 : cephadm 1 Reconfiguring daemon osd.1 on smithi148 2024-03-07T03:19:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: cluster 2024-03-07T03:19:46.251808+0000 mgr.y (mgr.34101) 114 : cluster 0 pgmap v205: 105 pgs: 105 active+clean; 285 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 436 op/s 2024-03-07T03:19:47.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:46 smithi195 bash[66542]: cluster 2024-03-07T03:19:46.654929+0000 mon.a (mon.0) 383 : cluster 0 osdmap e412: 8 total, 8 up, 8 in 2024-03-07T03:19:47.470 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:19:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:19:47] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2024-03-07T03:19:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: cluster 2024-03-07T03:19:47.660030+0000 mon.a (mon.0) 384 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2024-03-07T03:19:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: audit 2024-03-07T03:19:48.066648+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: audit 2024-03-07T03:19:48.072792+0000 mon.a (mon.0) 386 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: cephadm 2024-03-07T03:19:48.094206+0000 mgr.y (mgr.34101) 115 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2024-03-07T03:19:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: audit 2024-03-07T03:19:48.094841+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:19:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: audit 2024-03-07T03:19:48.095887+0000 mon.a (mon.0) 388 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: cephadm 2024-03-07T03:19:48.098700+0000 mgr.y (mgr.34101) 116 : cephadm 1 Reconfiguring daemon osd.2 on smithi148 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[79819]: cluster 2024-03-07T03:19:48.253122+0000 mgr.y (mgr.34101) 117 : cluster 0 pgmap v208: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 16 MiB/s wr, 370 op/s 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: cluster 2024-03-07T03:19:47.660030+0000 mon.a (mon.0) 384 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: audit 2024-03-07T03:19:48.066648+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: audit 2024-03-07T03:19:48.072792+0000 mon.a (mon.0) 386 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: cephadm 2024-03-07T03:19:48.094206+0000 mgr.y (mgr.34101) 115 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: audit 2024-03-07T03:19:48.094841+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: audit 2024-03-07T03:19:48.095887+0000 mon.a (mon.0) 388 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: cephadm 2024-03-07T03:19:48.098700+0000 mgr.y (mgr.34101) 116 : cephadm 1 Reconfiguring daemon osd.2 on smithi148 2024-03-07T03:19:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:48 smithi148 bash[82222]: cluster 2024-03-07T03:19:48.253122+0000 mgr.y (mgr.34101) 117 : cluster 0 pgmap v208: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 16 MiB/s wr, 370 op/s 2024-03-07T03:19:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: cluster 2024-03-07T03:19:47.660030+0000 mon.a (mon.0) 384 : cluster 0 osdmap e413: 8 total, 8 up, 8 in 2024-03-07T03:19:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: audit 2024-03-07T03:19:48.066648+0000 mon.a (mon.0) 385 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: audit 2024-03-07T03:19:48.072792+0000 mon.a (mon.0) 386 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: cephadm 2024-03-07T03:19:48.094206+0000 mgr.y (mgr.34101) 115 : cephadm 1 Reconfiguring osd.2 (monmap changed)... 2024-03-07T03:19:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: audit 2024-03-07T03:19:48.094841+0000 mon.a (mon.0) 387 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:19:49.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: audit 2024-03-07T03:19:48.095887+0000 mon.a (mon.0) 388 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:49.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: cephadm 2024-03-07T03:19:48.098700+0000 mgr.y (mgr.34101) 116 : cephadm 1 Reconfiguring daemon osd.2 on smithi148 2024-03-07T03:19:49.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:48 smithi195 bash[66542]: cluster 2024-03-07T03:19:48.253122+0000 mgr.y (mgr.34101) 117 : cluster 0 pgmap v208: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 MiB/s rd, 16 MiB/s wr, 370 op/s 2024-03-07T03:19:49.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:49 smithi148 bash[79819]: cluster 2024-03-07T03:19:48.667153+0000 mon.a (mon.0) 389 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2024-03-07T03:19:49.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:49 smithi148 bash[82222]: cluster 2024-03-07T03:19:48.667153+0000 mon.a (mon.0) 389 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2024-03-07T03:19:50.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:49 smithi195 bash[66542]: cluster 2024-03-07T03:19:48.667153+0000 mon.a (mon.0) 389 : cluster 0 osdmap e414: 8 total, 8 up, 8 in 2024-03-07T03:19:50.142 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (4516 ms) 2024-03-07T03:19:50.142 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: cluster 2024-03-07T03:19:49.671352+0000 mon.a (mon.0) 390 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: audit 2024-03-07T03:19:50.174936+0000 mon.a (mon.0) 391 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: audit 2024-03-07T03:19:50.181414+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: cephadm 2024-03-07T03:19:50.217817+0000 mgr.y (mgr.34101) 118 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: audit 2024-03-07T03:19:50.218255+0000 mon.a (mon.0) 393 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: audit 2024-03-07T03:19:50.219316+0000 mon.a (mon.0) 394 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: cephadm 2024-03-07T03:19:50.222972+0000 mgr.y (mgr.34101) 119 : cephadm 1 Reconfiguring daemon osd.3 on smithi148 2024-03-07T03:19:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[79819]: cluster 2024-03-07T03:19:50.253614+0000 mgr.y (mgr.34101) 120 : cluster 0 pgmap v211: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 18 MiB/s wr, 431 op/s 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: cluster 2024-03-07T03:19:49.671352+0000 mon.a (mon.0) 390 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: audit 2024-03-07T03:19:50.174936+0000 mon.a (mon.0) 391 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: audit 2024-03-07T03:19:50.181414+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: cephadm 2024-03-07T03:19:50.217817+0000 mgr.y (mgr.34101) 118 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: audit 2024-03-07T03:19:50.218255+0000 mon.a (mon.0) 393 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: audit 2024-03-07T03:19:50.219316+0000 mon.a (mon.0) 394 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: cephadm 2024-03-07T03:19:50.222972+0000 mgr.y (mgr.34101) 119 : cephadm 1 Reconfiguring daemon osd.3 on smithi148 2024-03-07T03:19:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:50 smithi148 bash[82222]: cluster 2024-03-07T03:19:50.253614+0000 mgr.y (mgr.34101) 120 : cluster 0 pgmap v211: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 18 MiB/s wr, 431 op/s 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: cluster 2024-03-07T03:19:49.671352+0000 mon.a (mon.0) 390 : cluster 0 osdmap e415: 8 total, 8 up, 8 in 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: audit 2024-03-07T03:19:50.174936+0000 mon.a (mon.0) 391 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: audit 2024-03-07T03:19:50.181414+0000 mon.a (mon.0) 392 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: cephadm 2024-03-07T03:19:50.217817+0000 mgr.y (mgr.34101) 118 : cephadm 1 Reconfiguring osd.3 (monmap changed)... 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: audit 2024-03-07T03:19:50.218255+0000 mon.a (mon.0) 393 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: audit 2024-03-07T03:19:50.219316+0000 mon.a (mon.0) 394 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: cephadm 2024-03-07T03:19:50.222972+0000 mgr.y (mgr.34101) 119 : cephadm 1 Reconfiguring daemon osd.3 on smithi148 2024-03-07T03:19:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:50 smithi195 bash[66542]: cluster 2024-03-07T03:19:50.253614+0000 mgr.y (mgr.34101) 120 : cluster 0 pgmap v211: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 31 MiB/s rd, 18 MiB/s wr, 431 op/s 2024-03-07T03:19:52.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:52 smithi148 bash[79819]: cluster 2024-03-07T03:19:51.194035+0000 mon.a (mon.0) 395 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2024-03-07T03:19:52.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:52 smithi148 bash[82222]: cluster 2024-03-07T03:19:51.194035+0000 mon.a (mon.0) 395 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2024-03-07T03:19:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:52 smithi195 bash[66542]: cluster 2024-03-07T03:19:51.194035+0000 mon.a (mon.0) 395 : cluster 0 osdmap e416: 8 total, 8 up, 8 in 2024-03-07T03:19:52.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:52 smithi195 bash[59134]: ts=2024-03-07T03:19:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:52.771 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2629 ms) 2024-03-07T03:19:52.771 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-03-07T03:19:53.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: cluster 2024-03-07T03:19:52.202154+0000 mon.a (mon.0) 396 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2024-03-07T03:19:53.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: cluster 2024-03-07T03:19:52.254317+0000 mgr.y (mgr.34101) 121 : cluster 0 pgmap v214: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:19:53.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: audit 2024-03-07T03:19:52.487919+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:53.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: audit 2024-03-07T03:19:52.494780+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:53.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: cephadm 2024-03-07T03:19:52.541449+0000 mgr.y (mgr.34101) 122 : cephadm 1 Reconfiguring mds.foo.smithi148.qizikc (monmap changed)... 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: audit 2024-03-07T03:19:52.541984+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: audit 2024-03-07T03:19:52.543246+0000 mon.a (mon.0) 400 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[79819]: cephadm 2024-03-07T03:19:52.544504+0000 mgr.y (mgr.34101) 123 : cephadm 1 Reconfiguring daemon mds.foo.smithi148.qizikc on smithi148 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: cluster 2024-03-07T03:19:52.202154+0000 mon.a (mon.0) 396 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: cluster 2024-03-07T03:19:52.254317+0000 mgr.y (mgr.34101) 121 : cluster 0 pgmap v214: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: audit 2024-03-07T03:19:52.487919+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: audit 2024-03-07T03:19:52.494780+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: cephadm 2024-03-07T03:19:52.541449+0000 mgr.y (mgr.34101) 122 : cephadm 1 Reconfiguring mds.foo.smithi148.qizikc (monmap changed)... 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: audit 2024-03-07T03:19:52.541984+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: audit 2024-03-07T03:19:52.543246+0000 mon.a (mon.0) 400 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:53.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:53 smithi148 bash[82222]: cephadm 2024-03-07T03:19:52.544504+0000 mgr.y (mgr.34101) 123 : cephadm 1 Reconfiguring daemon mds.foo.smithi148.qizikc on smithi148 2024-03-07T03:19:53.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: cluster 2024-03-07T03:19:52.202154+0000 mon.a (mon.0) 396 : cluster 0 osdmap e417: 8 total, 8 up, 8 in 2024-03-07T03:19:53.498 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: cluster 2024-03-07T03:19:52.254317+0000 mgr.y (mgr.34101) 121 : cluster 0 pgmap v214: 105 pgs: 105 active+clean; 361 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:19:53.498 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: audit 2024-03-07T03:19:52.487919+0000 mon.a (mon.0) 397 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:53.498 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: audit 2024-03-07T03:19:52.494780+0000 mon.a (mon.0) 398 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:53.498 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: cephadm 2024-03-07T03:19:52.541449+0000 mgr.y (mgr.34101) 122 : cephadm 1 Reconfiguring mds.foo.smithi148.qizikc (monmap changed)... 2024-03-07T03:19:53.498 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: audit 2024-03-07T03:19:52.541984+0000 mon.a (mon.0) 399 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi148.qizikc", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:19:53.498 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: audit 2024-03-07T03:19:52.543246+0000 mon.a (mon.0) 400 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:53.498 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[66542]: cephadm 2024-03-07T03:19:52.544504+0000 mgr.y (mgr.34101) 123 : cephadm 1 Reconfiguring daemon mds.foo.smithi148.qizikc on smithi148 2024-03-07T03:19:53.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:19:53 smithi195 bash[59134]: ts=2024-03-07T03:19:53.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:19:54.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:54 smithi148 bash[79819]: cluster 2024-03-07T03:19:53.501176+0000 mon.a (mon.0) 401 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2024-03-07T03:19:54.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:54 smithi148 bash[82222]: cluster 2024-03-07T03:19:53.501176+0000 mon.a (mon.0) 401 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2024-03-07T03:19:54.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:54 smithi195 bash[66542]: cluster 2024-03-07T03:19:53.501176+0000 mon.a (mon.0) 401 : cluster 0 osdmap e418: 8 total, 8 up, 8 in 2024-03-07T03:19:55.234 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2463 ms) 2024-03-07T03:19:55.234 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-03-07T03:19:55.285 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 65912081~2142795 2024-03-07T03:19:55.301 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 81210016~603153 2024-03-07T03:19:55.302 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 41038795~5578977 2024-03-07T03:19:55.324 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 2420393~344909 2024-03-07T03:19:55.325 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 77326993~5172413 2024-03-07T03:19:55.346 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 37303601~3959304 2024-03-07T03:19:55.349 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 40295088~4617347 2024-03-07T03:19:55.365 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 11870729~4530107 2024-03-07T03:19:55.365 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap0 2024-03-07T03:19:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:55 smithi195 bash[66542]: cluster 2024-03-07T03:19:54.255023+0000 mgr.y (mgr.34101) 124 : cluster 0 pgmap v216: 105 pgs: 105 active+clean; 437 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 308 op/s 2024-03-07T03:19:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:55 smithi195 bash[66542]: cluster 2024-03-07T03:19:54.513525+0000 mon.a (mon.0) 402 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2024-03-07T03:19:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:55 smithi195 bash[66542]: audit 2024-03-07T03:19:54.795328+0000 mon.a (mon.0) 403 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:55 smithi195 bash[66542]: audit 2024-03-07T03:19:54.804766+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:55.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:55 smithi195 bash[66542]: audit 2024-03-07T03:19:54.814971+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:55.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:55 smithi195 bash[66542]: audit 2024-03-07T03:19:54.816817+0000 mon.a (mon.0) 406 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:55.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:55 smithi195 bash[66542]: audit 2024-03-07T03:19:54.818019+0000 mon.a (mon.0) 407 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[79819]: cluster 2024-03-07T03:19:54.255023+0000 mgr.y (mgr.34101) 124 : cluster 0 pgmap v216: 105 pgs: 105 active+clean; 437 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 308 op/s 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[79819]: cluster 2024-03-07T03:19:54.513525+0000 mon.a (mon.0) 402 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[79819]: audit 2024-03-07T03:19:54.795328+0000 mon.a (mon.0) 403 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[79819]: audit 2024-03-07T03:19:54.804766+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[79819]: audit 2024-03-07T03:19:54.814971+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[79819]: audit 2024-03-07T03:19:54.816817+0000 mon.a (mon.0) 406 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[79819]: audit 2024-03-07T03:19:54.818019+0000 mon.a (mon.0) 407 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:55.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[82222]: cluster 2024-03-07T03:19:54.255023+0000 mgr.y (mgr.34101) 124 : cluster 0 pgmap v216: 105 pgs: 105 active+clean; 437 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 308 op/s 2024-03-07T03:19:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[82222]: cluster 2024-03-07T03:19:54.513525+0000 mon.a (mon.0) 402 : cluster 0 osdmap e419: 8 total, 8 up, 8 in 2024-03-07T03:19:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[82222]: audit 2024-03-07T03:19:54.795328+0000 mon.a (mon.0) 403 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[82222]: audit 2024-03-07T03:19:54.804766+0000 mon.a (mon.0) 404 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[82222]: audit 2024-03-07T03:19:54.814971+0000 mon.a (mon.0) 405 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-07T03:19:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[82222]: audit 2024-03-07T03:19:54.816817+0000 mon.a (mon.0) 406 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-07T03:19:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:55 smithi148 bash[82222]: audit 2024-03-07T03:19:54.818019+0000 mon.a (mon.0) 407 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:55.836 INFO:tasks.workunit.client.0.smithi148.stdout:clone image260 -> image261 2024-03-07T03:19:55.975 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 13871356~6201478 2024-03-07T03:19:56.015 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 53245095~2790322 2024-03-07T03:19:56.024 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 80107343~3778737 2024-03-07T03:19:56.061 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 43776612~2618584 2024-03-07T03:19:56.083 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 62393204~1747449 2024-03-07T03:19:56.103 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 54774745~4303086 2024-03-07T03:19:56.109 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 65166137~8289577 2024-03-07T03:19:56.143 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 34058554~3246314 2024-03-07T03:19:56.149 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:19:56.756 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:56 smithi195 bash[66542]: cephadm 2024-03-07T03:19:54.814332+0000 mgr.y (mgr.34101) 125 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:19:56.756 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:56 smithi195 bash[66542]: cephadm 2024-03-07T03:19:54.819156+0000 mgr.y (mgr.34101) 126 : cephadm 1 Reconfiguring daemon mon.b on smithi195 2024-03-07T03:19:56.756 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:56 smithi195 bash[66542]: cluster 2024-03-07T03:19:55.811324+0000 mon.a (mon.0) 408 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2024-03-07T03:19:56.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:56 smithi148 bash[79819]: cephadm 2024-03-07T03:19:54.814332+0000 mgr.y (mgr.34101) 125 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:19:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:56 smithi148 bash[79819]: cephadm 2024-03-07T03:19:54.819156+0000 mgr.y (mgr.34101) 126 : cephadm 1 Reconfiguring daemon mon.b on smithi195 2024-03-07T03:19:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:56 smithi148 bash[79819]: cluster 2024-03-07T03:19:55.811324+0000 mon.a (mon.0) 408 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2024-03-07T03:19:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:56 smithi148 bash[82222]: cephadm 2024-03-07T03:19:54.814332+0000 mgr.y (mgr.34101) 125 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-03-07T03:19:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:56 smithi148 bash[82222]: cephadm 2024-03-07T03:19:54.819156+0000 mgr.y (mgr.34101) 126 : cephadm 1 Reconfiguring daemon mon.b on smithi195 2024-03-07T03:19:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:56 smithi148 bash[82222]: cluster 2024-03-07T03:19:55.811324+0000 mon.a (mon.0) 408 : cluster 0 osdmap e420: 8 total, 8 up, 8 in 2024-03-07T03:19:56.845 INFO:tasks.workunit.client.0.smithi148.stdout:clone image261 -> image262 2024-03-07T03:19:57.011 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 158983429 2024-03-07T03:19:57.026 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 154868703~4114726 2024-03-07T03:19:57.057 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 153352059~4772290 2024-03-07T03:19:57.063 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 109985002~6816381 2024-03-07T03:19:57.095 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 60701676~2172986 2024-03-07T03:19:57.105 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 149061690~4662959 2024-03-07T03:19:57.121 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 72867448~780248 2024-03-07T03:19:57.143 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 60995997~3467978 2024-03-07T03:19:57.178 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 98656113~3347308 2024-03-07T03:19:57.179 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:19:57.496 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:19:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:19:57] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.43.0" 2024-03-07T03:19:57.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:57 smithi148 bash[79819]: cluster 2024-03-07T03:19:56.255821+0000 mgr.y (mgr.34101) 127 : cluster 0 pgmap v219: 105 pgs: 105 active+clean; 437 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 348 op/s 2024-03-07T03:19:57.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:57 smithi148 bash[79819]: cluster 2024-03-07T03:19:56.820539+0000 mon.a (mon.0) 409 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2024-03-07T03:19:57.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:57 smithi148 bash[79819]: audit 2024-03-07T03:19:57.301591+0000 mon.a (mon.0) 410 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:57.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:57 smithi148 bash[82222]: cluster 2024-03-07T03:19:56.255821+0000 mgr.y (mgr.34101) 127 : cluster 0 pgmap v219: 105 pgs: 105 active+clean; 437 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 348 op/s 2024-03-07T03:19:57.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:57 smithi148 bash[82222]: cluster 2024-03-07T03:19:56.820539+0000 mon.a (mon.0) 409 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2024-03-07T03:19:57.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:57 smithi148 bash[82222]: audit 2024-03-07T03:19:57.301591+0000 mon.a (mon.0) 410 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:57 smithi195 bash[66542]: cluster 2024-03-07T03:19:56.255821+0000 mgr.y (mgr.34101) 127 : cluster 0 pgmap v219: 105 pgs: 105 active+clean; 437 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 40 MiB/s rd, 19 MiB/s wr, 348 op/s 2024-03-07T03:19:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:57 smithi195 bash[66542]: cluster 2024-03-07T03:19:56.820539+0000 mon.a (mon.0) 409 : cluster 0 osdmap e421: 8 total, 8 up, 8 in 2024-03-07T03:19:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:57 smithi195 bash[66542]: audit 2024-03-07T03:19:57.301591+0000 mon.a (mon.0) 410 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:19:57.848 INFO:tasks.workunit.client.0.smithi148.stdout:clone image262 -> image263 2024-03-07T03:19:58.037 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 65407938~611401 2024-03-07T03:19:58.078 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 59417473~4515252 2024-03-07T03:19:58.092 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 36124788~7789784 2024-03-07T03:19:58.126 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 149253639~4185318 2024-03-07T03:19:58.143 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 84188143~3816056 2024-03-07T03:19:58.167 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 32219424~1228266 2024-03-07T03:19:58.179 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 6543361~4289318 2024-03-07T03:19:58.197 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 95348427~7109816 2024-03-07T03:19:58.207 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap3 2024-03-07T03:19:58.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: audit 2024-03-07T03:19:57.524219+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:58.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: audit 2024-03-07T03:19:57.537852+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:58.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: cephadm 2024-03-07T03:19:57.546737+0000 mgr.y (mgr.34101) 128 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2024-03-07T03:19:58.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: audit 2024-03-07T03:19:57.547497+0000 mon.a (mon.0) 413 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:19:58.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: audit 2024-03-07T03:19:57.549568+0000 mon.a (mon.0) 414 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: audit 2024-03-07T03:19:57.551289+0000 mon.a (mon.0) 415 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: cephadm 2024-03-07T03:19:57.553357+0000 mgr.y (mgr.34101) 129 : cephadm 1 Reconfiguring daemon mgr.x on smithi195 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[79819]: cluster 2024-03-07T03:19:57.824313+0000 mon.a (mon.0) 416 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: audit 2024-03-07T03:19:57.524219+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: audit 2024-03-07T03:19:57.537852+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: cephadm 2024-03-07T03:19:57.546737+0000 mgr.y (mgr.34101) 128 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: audit 2024-03-07T03:19:57.547497+0000 mon.a (mon.0) 413 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: audit 2024-03-07T03:19:57.549568+0000 mon.a (mon.0) 414 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: audit 2024-03-07T03:19:57.551289+0000 mon.a (mon.0) 415 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: cephadm 2024-03-07T03:19:57.553357+0000 mgr.y (mgr.34101) 129 : cephadm 1 Reconfiguring daemon mgr.x on smithi195 2024-03-07T03:19:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:58 smithi148 bash[82222]: cluster 2024-03-07T03:19:57.824313+0000 mon.a (mon.0) 416 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2024-03-07T03:19:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: audit 2024-03-07T03:19:57.524219+0000 mon.a (mon.0) 411 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: audit 2024-03-07T03:19:57.537852+0000 mon.a (mon.0) 412 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:19:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: cephadm 2024-03-07T03:19:57.546737+0000 mgr.y (mgr.34101) 128 : cephadm 1 Reconfiguring mgr.x (monmap changed)... 2024-03-07T03:19:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: audit 2024-03-07T03:19:57.547497+0000 mon.a (mon.0) 413 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-07T03:19:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: audit 2024-03-07T03:19:57.549568+0000 mon.a (mon.0) 414 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-07T03:19:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: audit 2024-03-07T03:19:57.551289+0000 mon.a (mon.0) 415 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:19:58.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: cephadm 2024-03-07T03:19:57.553357+0000 mgr.y (mgr.34101) 129 : cephadm 1 Reconfiguring daemon mgr.x on smithi195 2024-03-07T03:19:58.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:58 smithi195 bash[66542]: cluster 2024-03-07T03:19:57.824313+0000 mon.a (mon.0) 416 : cluster 0 osdmap e422: 8 total, 8 up, 8 in 2024-03-07T03:19:58.855 INFO:tasks.workunit.client.0.smithi148.stdout:clone image263 -> image264 2024-03-07T03:19:59.756 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (4522 ms) 2024-03-07T03:19:59.756 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-03-07T03:19:59.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:59 smithi148 bash[79819]: cluster 2024-03-07T03:19:58.257171+0000 mgr.y (mgr.34101) 130 : cluster 0 pgmap v222: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 43 MiB/s wr, 565 op/s 2024-03-07T03:19:59.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:19:59 smithi148 bash[79819]: cluster 2024-03-07T03:19:58.830589+0000 mon.a (mon.0) 417 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2024-03-07T03:19:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:59 smithi148 bash[82222]: cluster 2024-03-07T03:19:58.257171+0000 mgr.y (mgr.34101) 130 : cluster 0 pgmap v222: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 43 MiB/s wr, 565 op/s 2024-03-07T03:19:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:19:59 smithi148 bash[82222]: cluster 2024-03-07T03:19:58.830589+0000 mon.a (mon.0) 417 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2024-03-07T03:19:59.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:59 smithi195 bash[66542]: cluster 2024-03-07T03:19:58.257171+0000 mgr.y (mgr.34101) 130 : cluster 0 pgmap v222: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 43 MiB/s wr, 565 op/s 2024-03-07T03:19:59.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:19:59 smithi195 bash[66542]: cluster 2024-03-07T03:19:58.830589+0000 mon.a (mon.0) 417 : cluster 0 osdmap e423: 8 total, 8 up, 8 in 2024-03-07T03:20:00.138 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (382 ms) 2024-03-07T03:20:00.138 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-03-07T03:20:00.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[79819]: debug 2024-03-07T03:19:59.999+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[79819]: cluster 2024-03-07T03:20:00.000311+0000 mon.a (mon.0) 418 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[79819]: audit 2024-03-07T03:20:00.044323+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[79819]: audit 2024-03-07T03:20:00.055869+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[79819]: audit 2024-03-07T03:20:00.073626+0000 mon.a (mon.0) 421 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:20:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[79819]: audit 2024-03-07T03:20:00.075198+0000 mon.a (mon.0) 422 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[82222]: cluster 2024-03-07T03:20:00.000311+0000 mon.a (mon.0) 418 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[82222]: audit 2024-03-07T03:20:00.044323+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[82222]: audit 2024-03-07T03:20:00.055869+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[82222]: audit 2024-03-07T03:20:00.073626+0000 mon.a (mon.0) 421 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:20:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:00 smithi148 bash[82222]: audit 2024-03-07T03:20:00.075198+0000 mon.a (mon.0) 422 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:00 smithi195 bash[66542]: cluster 2024-03-07T03:20:00.000311+0000 mon.a (mon.0) 418 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:20:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:00 smithi195 bash[66542]: audit 2024-03-07T03:20:00.044323+0000 mon.a (mon.0) 419 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:00 smithi195 bash[66542]: audit 2024-03-07T03:20:00.055869+0000 mon.a (mon.0) 420 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:00 smithi195 bash[66542]: audit 2024-03-07T03:20:00.073626+0000 mon.a (mon.0) 421 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:20:00.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:00 smithi195 bash[66542]: audit 2024-03-07T03:20:00.075198+0000 mon.a (mon.0) 422 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[82222]: cephadm 2024-03-07T03:20:00.072949+0000 mgr.y (mgr.34101) 131 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2024-03-07T03:20:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[82222]: cephadm 2024-03-07T03:20:00.080106+0000 mgr.y (mgr.34101) 132 : cephadm 1 Reconfiguring daemon osd.4 on smithi195 2024-03-07T03:20:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[82222]: cluster 2024-03-07T03:20:00.257842+0000 mgr.y (mgr.34101) 133 : cluster 0 pgmap v224: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 39 MiB/s wr, 508 op/s 2024-03-07T03:20:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[82222]: cluster 2024-03-07T03:20:01.069283+0000 mon.a (mon.0) 423 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2024-03-07T03:20:01.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[79819]: cephadm 2024-03-07T03:20:00.072949+0000 mgr.y (mgr.34101) 131 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2024-03-07T03:20:01.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[79819]: cephadm 2024-03-07T03:20:00.080106+0000 mgr.y (mgr.34101) 132 : cephadm 1 Reconfiguring daemon osd.4 on smithi195 2024-03-07T03:20:01.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[79819]: cluster 2024-03-07T03:20:00.257842+0000 mgr.y (mgr.34101) 133 : cluster 0 pgmap v224: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 39 MiB/s wr, 508 op/s 2024-03-07T03:20:01.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:01 smithi148 bash[79819]: cluster 2024-03-07T03:20:01.069283+0000 mon.a (mon.0) 423 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2024-03-07T03:20:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:01 smithi195 bash[66542]: cephadm 2024-03-07T03:20:00.072949+0000 mgr.y (mgr.34101) 131 : cephadm 1 Reconfiguring osd.4 (monmap changed)... 2024-03-07T03:20:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:01 smithi195 bash[66542]: cephadm 2024-03-07T03:20:00.080106+0000 mgr.y (mgr.34101) 132 : cephadm 1 Reconfiguring daemon osd.4 on smithi195 2024-03-07T03:20:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:01 smithi195 bash[66542]: cluster 2024-03-07T03:20:00.257842+0000 mgr.y (mgr.34101) 133 : cluster 0 pgmap v224: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 39 MiB/s wr, 508 op/s 2024-03-07T03:20:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:01 smithi195 bash[66542]: cluster 2024-03-07T03:20:01.069283+0000 mon.a (mon.0) 423 : cluster 0 osdmap e424: 8 total, 8 up, 8 in 2024-03-07T03:20:02.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:02 smithi195 bash[59134]: ts=2024-03-07T03:20:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:03.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: cluster 2024-03-07T03:20:02.073134+0000 mon.a (mon.0) 424 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2024-03-07T03:20:03.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: cluster 2024-03-07T03:20:02.258531+0000 mgr.y (mgr.34101) 134 : cluster 0 pgmap v227: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 39 MiB/s wr, 509 op/s 2024-03-07T03:20:03.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: audit 2024-03-07T03:20:02.419028+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:03.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: audit 2024-03-07T03:20:02.432386+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:03.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: cephadm 2024-03-07T03:20:02.451037+0000 mgr.y (mgr.34101) 135 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2024-03-07T03:20:03.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: audit 2024-03-07T03:20:02.451743+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:20:03.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: audit 2024-03-07T03:20:02.453334+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:03.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[66542]: cephadm 2024-03-07T03:20:02.458503+0000 mgr.y (mgr.34101) 136 : cephadm 1 Reconfiguring daemon osd.5 on smithi195 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: cluster 2024-03-07T03:20:02.073134+0000 mon.a (mon.0) 424 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: cluster 2024-03-07T03:20:02.258531+0000 mgr.y (mgr.34101) 134 : cluster 0 pgmap v227: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 39 MiB/s wr, 509 op/s 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: audit 2024-03-07T03:20:02.419028+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: audit 2024-03-07T03:20:02.432386+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: cephadm 2024-03-07T03:20:02.451037+0000 mgr.y (mgr.34101) 135 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: audit 2024-03-07T03:20:02.451743+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: audit 2024-03-07T03:20:02.453334+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[79819]: cephadm 2024-03-07T03:20:02.458503+0000 mgr.y (mgr.34101) 136 : cephadm 1 Reconfiguring daemon osd.5 on smithi195 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: cluster 2024-03-07T03:20:02.073134+0000 mon.a (mon.0) 424 : cluster 0 osdmap e425: 8 total, 8 up, 8 in 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: cluster 2024-03-07T03:20:02.258531+0000 mgr.y (mgr.34101) 134 : cluster 0 pgmap v227: 105 pgs: 105 active+clean; 639 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 39 MiB/s wr, 509 op/s 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: audit 2024-03-07T03:20:02.419028+0000 mon.a (mon.0) 425 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: audit 2024-03-07T03:20:02.432386+0000 mon.a (mon.0) 426 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: cephadm 2024-03-07T03:20:02.451037+0000 mgr.y (mgr.34101) 135 : cephadm 1 Reconfiguring osd.5 (monmap changed)... 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: audit 2024-03-07T03:20:02.451743+0000 mon.a (mon.0) 427 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: audit 2024-03-07T03:20:02.453334+0000 mon.a (mon.0) 428 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:03 smithi148 bash[82222]: cephadm 2024-03-07T03:20:02.458503+0000 mgr.y (mgr.34101) 136 : cephadm 1 Reconfiguring daemon osd.5 on smithi195 2024-03-07T03:20:03.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:03 smithi195 bash[59134]: ts=2024-03-07T03:20:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:04.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:04 smithi195 bash[66542]: cluster 2024-03-07T03:20:03.086073+0000 mon.a (mon.0) 429 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2024-03-07T03:20:04.493 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (4355 ms) 2024-03-07T03:20:04.493 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-03-07T03:20:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:04 smithi148 bash[79819]: cluster 2024-03-07T03:20:03.086073+0000 mon.a (mon.0) 429 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2024-03-07T03:20:04.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:04 smithi148 bash[82222]: cluster 2024-03-07T03:20:03.086073+0000 mon.a (mon.0) 429 : cluster 0 osdmap e426: 8 total, 8 up, 8 in 2024-03-07T03:20:05.478 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:05 smithi195 bash[66542]: cluster 2024-03-07T03:20:04.085488+0000 mon.a (mon.0) 430 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2024-03-07T03:20:05.478 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:05 smithi195 bash[66542]: cluster 2024-03-07T03:20:04.259195+0000 mgr.y (mgr.34101) 137 : cluster 0 pgmap v230: 105 pgs: 105 active+clean; 687 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 12 MiB/s wr, 421 op/s 2024-03-07T03:20:05.478 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:05 smithi195 bash[66542]: audit 2024-03-07T03:20:05.009648+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:05.478 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:05 smithi195 bash[66542]: audit 2024-03-07T03:20:05.021098+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:05.478 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:05 smithi195 bash[66542]: audit 2024-03-07T03:20:05.029847+0000 mon.a (mon.0) 433 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:20:05.478 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:05 smithi195 bash[66542]: audit 2024-03-07T03:20:05.031647+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[79819]: cluster 2024-03-07T03:20:04.085488+0000 mon.a (mon.0) 430 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[79819]: cluster 2024-03-07T03:20:04.259195+0000 mgr.y (mgr.34101) 137 : cluster 0 pgmap v230: 105 pgs: 105 active+clean; 687 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 12 MiB/s wr, 421 op/s 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[79819]: audit 2024-03-07T03:20:05.009648+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[79819]: audit 2024-03-07T03:20:05.021098+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[79819]: audit 2024-03-07T03:20:05.029847+0000 mon.a (mon.0) 433 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[79819]: audit 2024-03-07T03:20:05.031647+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[82222]: cluster 2024-03-07T03:20:04.085488+0000 mon.a (mon.0) 430 : cluster 0 osdmap e427: 8 total, 8 up, 8 in 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[82222]: cluster 2024-03-07T03:20:04.259195+0000 mgr.y (mgr.34101) 137 : cluster 0 pgmap v230: 105 pgs: 105 active+clean; 687 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 12 MiB/s wr, 421 op/s 2024-03-07T03:20:05.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[82222]: audit 2024-03-07T03:20:05.009648+0000 mon.a (mon.0) 431 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:05.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[82222]: audit 2024-03-07T03:20:05.021098+0000 mon.a (mon.0) 432 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:05.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[82222]: audit 2024-03-07T03:20:05.029847+0000 mon.a (mon.0) 433 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-07T03:20:05.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:05 smithi148 bash[82222]: audit 2024-03-07T03:20:05.031647+0000 mon.a (mon.0) 434 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:06 smithi195 bash[66542]: cephadm 2024-03-07T03:20:05.029151+0000 mgr.y (mgr.34101) 138 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2024-03-07T03:20:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:06 smithi195 bash[66542]: cephadm 2024-03-07T03:20:05.037023+0000 mgr.y (mgr.34101) 139 : cephadm 1 Reconfiguring daemon osd.6 on smithi195 2024-03-07T03:20:06.531 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:06 smithi195 bash[66542]: cluster 2024-03-07T03:20:05.106064+0000 mon.a (mon.0) 435 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2024-03-07T03:20:06.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:06 smithi148 bash[79819]: cephadm 2024-03-07T03:20:05.029151+0000 mgr.y (mgr.34101) 138 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2024-03-07T03:20:06.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:06 smithi148 bash[79819]: cephadm 2024-03-07T03:20:05.037023+0000 mgr.y (mgr.34101) 139 : cephadm 1 Reconfiguring daemon osd.6 on smithi195 2024-03-07T03:20:06.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:06 smithi148 bash[79819]: cluster 2024-03-07T03:20:05.106064+0000 mon.a (mon.0) 435 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2024-03-07T03:20:06.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:06 smithi148 bash[82222]: cephadm 2024-03-07T03:20:05.029151+0000 mgr.y (mgr.34101) 138 : cephadm 1 Reconfiguring osd.6 (monmap changed)... 2024-03-07T03:20:06.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:06 smithi148 bash[82222]: cephadm 2024-03-07T03:20:05.037023+0000 mgr.y (mgr.34101) 139 : cephadm 1 Reconfiguring daemon osd.6 on smithi195 2024-03-07T03:20:06.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:06 smithi148 bash[82222]: cluster 2024-03-07T03:20:05.106064+0000 mon.a (mon.0) 435 : cluster 0 osdmap e428: 8 total, 8 up, 8 in 2024-03-07T03:20:06.760 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2268 ms) 2024-03-07T03:20:06.761 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-03-07T03:20:07.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:07 smithi195 bash[66542]: cluster 2024-03-07T03:20:06.115697+0000 mon.a (mon.0) 436 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2024-03-07T03:20:07.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:07 smithi195 bash[66542]: cluster 2024-03-07T03:20:06.259871+0000 mgr.y (mgr.34101) 140 : cluster 0 pgmap v233: 105 pgs: 105 active+clean; 687 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 12 MiB/s wr, 421 op/s 2024-03-07T03:20:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:07 smithi148 bash[79819]: cluster 2024-03-07T03:20:06.115697+0000 mon.a (mon.0) 436 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2024-03-07T03:20:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:07 smithi148 bash[79819]: cluster 2024-03-07T03:20:06.259871+0000 mgr.y (mgr.34101) 140 : cluster 0 pgmap v233: 105 pgs: 105 active+clean; 687 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 12 MiB/s wr, 421 op/s 2024-03-07T03:20:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:20:07] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2024-03-07T03:20:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:07 smithi148 bash[82222]: cluster 2024-03-07T03:20:06.115697+0000 mon.a (mon.0) 436 : cluster 0 osdmap e429: 8 total, 8 up, 8 in 2024-03-07T03:20:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:07 smithi148 bash[82222]: cluster 2024-03-07T03:20:06.259871+0000 mgr.y (mgr.34101) 140 : cluster 0 pgmap v233: 105 pgs: 105 active+clean; 687 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 34 MiB/s rd, 12 MiB/s wr, 421 op/s 2024-03-07T03:20:08.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:08 smithi195 bash[66542]: cluster 2024-03-07T03:20:07.129217+0000 mon.a (mon.0) 437 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2024-03-07T03:20:08.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:08 smithi195 bash[66542]: audit 2024-03-07T03:20:07.514744+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:08.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:08 smithi195 bash[66542]: audit 2024-03-07T03:20:07.526440+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:08.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:08 smithi195 bash[66542]: cephadm 2024-03-07T03:20:07.558261+0000 mgr.y (mgr.34101) 141 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2024-03-07T03:20:08.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:08 smithi195 bash[66542]: audit 2024-03-07T03:20:07.559016+0000 mon.a (mon.0) 440 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:20:08.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:08 smithi195 bash[66542]: audit 2024-03-07T03:20:07.561002+0000 mon.a (mon.0) 441 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:08.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:08 smithi195 bash[66542]: cephadm 2024-03-07T03:20:07.566422+0000 mgr.y (mgr.34101) 142 : cephadm 1 Reconfiguring daemon osd.7 on smithi195 2024-03-07T03:20:08.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[79819]: cluster 2024-03-07T03:20:07.129217+0000 mon.a (mon.0) 437 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[79819]: audit 2024-03-07T03:20:07.514744+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[79819]: audit 2024-03-07T03:20:07.526440+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[79819]: cephadm 2024-03-07T03:20:07.558261+0000 mgr.y (mgr.34101) 141 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[79819]: audit 2024-03-07T03:20:07.559016+0000 mon.a (mon.0) 440 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[79819]: audit 2024-03-07T03:20:07.561002+0000 mon.a (mon.0) 441 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[79819]: cephadm 2024-03-07T03:20:07.566422+0000 mgr.y (mgr.34101) 142 : cephadm 1 Reconfiguring daemon osd.7 on smithi195 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[82222]: cluster 2024-03-07T03:20:07.129217+0000 mon.a (mon.0) 437 : cluster 0 osdmap e430: 8 total, 8 up, 8 in 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[82222]: audit 2024-03-07T03:20:07.514744+0000 mon.a (mon.0) 438 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[82222]: audit 2024-03-07T03:20:07.526440+0000 mon.a (mon.0) 439 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:08.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[82222]: cephadm 2024-03-07T03:20:07.558261+0000 mgr.y (mgr.34101) 141 : cephadm 1 Reconfiguring osd.7 (monmap changed)... 2024-03-07T03:20:08.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[82222]: audit 2024-03-07T03:20:07.559016+0000 mon.a (mon.0) 440 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-07T03:20:08.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[82222]: audit 2024-03-07T03:20:07.561002+0000 mon.a (mon.0) 441 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:08.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:08 smithi148 bash[82222]: cephadm 2024-03-07T03:20:07.566422+0000 mgr.y (mgr.34101) 142 : cephadm 1 Reconfiguring daemon osd.7 on smithi195 2024-03-07T03:20:08.778 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2017 ms) 2024-03-07T03:20:08.778 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-03-07T03:20:08.808 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 27503941~5468812 2024-03-07T03:20:08.831 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 6025798~3097660 2024-03-07T03:20:08.831 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 34526480~5806095 2024-03-07T03:20:08.847 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 37353344~984498 2024-03-07T03:20:08.849 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 62472556~2625174 2024-03-07T03:20:08.859 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 14500480~4410313 2024-03-07T03:20:08.860 INFO:tasks.workunit.client.0.smithi148.stdout:write: A 50637702~8267552 2024-03-07T03:20:08.884 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 25614510~5513048 2024-03-07T03:20:08.886 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap0 2024-03-07T03:20:09.168 INFO:tasks.workunit.client.0.smithi148.stdout:clone image283 -> image284 2024-03-07T03:20:09.271 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 88150822 2024-03-07T03:20:09.282 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 36627187~792206 2024-03-07T03:20:09.312 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 63238276~128873 2024-03-07T03:20:09.335 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 82595286~1154147 2024-03-07T03:20:09.346 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 3911902~5754525 2024-03-07T03:20:09.359 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 1450380~4387672 2024-03-07T03:20:09.374 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 81639051~735788 2024-03-07T03:20:09.377 INFO:tasks.workunit.client.0.smithi148.stdout:write: B 51996793~3640583 2024-03-07T03:20:09.418 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 71946999~1414135 2024-03-07T03:20:09.418 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:09 smithi195 bash[66542]: cluster 2024-03-07T03:20:08.139140+0000 mon.a (mon.0) 442 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2024-03-07T03:20:09.419 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:09 smithi195 bash[66542]: cluster 2024-03-07T03:20:08.261183+0000 mgr.y (mgr.34101) 143 : cluster 0 pgmap v236: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 495 op/s 2024-03-07T03:20:09.424 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:20:09.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:09 smithi148 bash[79819]: cluster 2024-03-07T03:20:08.139140+0000 mon.a (mon.0) 442 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2024-03-07T03:20:09.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:09 smithi148 bash[79819]: cluster 2024-03-07T03:20:08.261183+0000 mgr.y (mgr.34101) 143 : cluster 0 pgmap v236: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 495 op/s 2024-03-07T03:20:09.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:09 smithi148 bash[82222]: cluster 2024-03-07T03:20:08.139140+0000 mon.a (mon.0) 442 : cluster 0 osdmap e431: 8 total, 8 up, 8 in 2024-03-07T03:20:09.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:09 smithi148 bash[82222]: cluster 2024-03-07T03:20:08.261183+0000 mgr.y (mgr.34101) 143 : cluster 0 pgmap v236: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 495 op/s 2024-03-07T03:20:10.177 INFO:tasks.workunit.client.0.smithi148.stdout:clone image284 -> image285 2024-03-07T03:20:10.342 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 28047369~2262271 2024-03-07T03:20:10.382 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 63185191~5011358 2024-03-07T03:20:10.397 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 52502767~312011 2024-03-07T03:20:10.434 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 58211313~5893516 2024-03-07T03:20:10.468 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 68200192~8247671 2024-03-07T03:20:10.501 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 73781539~5109438 2024-03-07T03:20:10.505 INFO:tasks.workunit.client.0.smithi148.stdout:write: C 9743758~2218336 2024-03-07T03:20:10.522 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 63923447~7030324 2024-03-07T03:20:10.524 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:20:10.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[82222]: cluster 2024-03-07T03:20:09.149024+0000 mon.a (mon.0) 443 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[82222]: audit 2024-03-07T03:20:10.064110+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[82222]: audit 2024-03-07T03:20:10.076410+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[82222]: audit 2024-03-07T03:20:10.089409+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[82222]: audit 2024-03-07T03:20:10.091287+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[79819]: cluster 2024-03-07T03:20:09.149024+0000 mon.a (mon.0) 443 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[79819]: audit 2024-03-07T03:20:10.064110+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[79819]: audit 2024-03-07T03:20:10.076410+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[79819]: audit 2024-03-07T03:20:10.089409+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:20:10.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:10 smithi148 bash[79819]: audit 2024-03-07T03:20:10.091287+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:10.548 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:20:10 smithi195 bash[49044]: logger=sqlstore.transactions t=2024-03-07T03:20:10.124283622Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-07T03:20:10.548 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:20:10 smithi195 bash[49044]: logger=cleanup t=2024-03-07T03:20:10.230728861Z level=info msg="Completed cleanup jobs" duration=117.830323ms 2024-03-07T03:20:10.548 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:10 smithi195 bash[66542]: cluster 2024-03-07T03:20:09.149024+0000 mon.a (mon.0) 443 : cluster 0 osdmap e432: 8 total, 8 up, 8 in 2024-03-07T03:20:10.548 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:10 smithi195 bash[66542]: audit 2024-03-07T03:20:10.064110+0000 mon.a (mon.0) 444 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:10.548 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:10 smithi195 bash[66542]: audit 2024-03-07T03:20:10.076410+0000 mon.a (mon.0) 445 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:10.548 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:10 smithi195 bash[66542]: audit 2024-03-07T03:20:10.089409+0000 mon.a (mon.0) 446 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi195.gtlmjn", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-07T03:20:10.548 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:10 smithi195 bash[66542]: audit 2024-03-07T03:20:10.091287+0000 mon.a (mon.0) 447 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:11.200 INFO:tasks.workunit.client.0.smithi148.stdout:clone image285 -> image286 2024-03-07T03:20:11.388 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 7644706~1197291 2024-03-07T03:20:11.436 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 30469035~3848642 2024-03-07T03:20:11.475 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 28482370~6520857 2024-03-07T03:20:11.504 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 33777628~3901439 2024-03-07T03:20:11.510 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 25089648~7658336 2024-03-07T03:20:11.536 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 7445349~3050605 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[79819]: cephadm 2024-03-07T03:20:10.088697+0000 mgr.y (mgr.34101) 144 : cephadm 1 Reconfiguring mds.foo.smithi195.gtlmjn (monmap changed)... 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[79819]: cephadm 2024-03-07T03:20:10.093169+0000 mgr.y (mgr.34101) 145 : cephadm 1 Reconfiguring daemon mds.foo.smithi195.gtlmjn on smithi195 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[79819]: cluster 2024-03-07T03:20:10.159142+0000 mon.a (mon.0) 448 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[79819]: cluster 2024-03-07T03:20:10.261867+0000 mgr.y (mgr.34101) 146 : cluster 0 pgmap v239: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 495 op/s 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[82222]: cephadm 2024-03-07T03:20:10.088697+0000 mgr.y (mgr.34101) 144 : cephadm 1 Reconfiguring mds.foo.smithi195.gtlmjn (monmap changed)... 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[82222]: cephadm 2024-03-07T03:20:10.093169+0000 mgr.y (mgr.34101) 145 : cephadm 1 Reconfiguring daemon mds.foo.smithi195.gtlmjn on smithi195 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[82222]: cluster 2024-03-07T03:20:10.159142+0000 mon.a (mon.0) 448 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2024-03-07T03:20:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:11 smithi148 bash[82222]: cluster 2024-03-07T03:20:10.261867+0000 mgr.y (mgr.34101) 146 : cluster 0 pgmap v239: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 495 op/s 2024-03-07T03:20:11.542 INFO:tasks.workunit.client.0.smithi148.stdout:write: D 80449064~7701758 2024-03-07T03:20:11.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:11 smithi195 bash[66542]: cephadm 2024-03-07T03:20:10.088697+0000 mgr.y (mgr.34101) 144 : cephadm 1 Reconfiguring mds.foo.smithi195.gtlmjn (monmap changed)... 2024-03-07T03:20:11.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:11 smithi195 bash[66542]: cephadm 2024-03-07T03:20:10.093169+0000 mgr.y (mgr.34101) 145 : cephadm 1 Reconfiguring daemon mds.foo.smithi195.gtlmjn on smithi195 2024-03-07T03:20:11.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:11 smithi195 bash[66542]: cluster 2024-03-07T03:20:10.159142+0000 mon.a (mon.0) 448 : cluster 0 osdmap e433: 8 total, 8 up, 8 in 2024-03-07T03:20:11.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:11 smithi195 bash[66542]: cluster 2024-03-07T03:20:10.261867+0000 mgr.y (mgr.34101) 146 : cluster 0 pgmap v239: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 18 MiB/s wr, 495 op/s 2024-03-07T03:20:11.575 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 7180491~4051426 2024-03-07T03:20:11.578 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap3 2024-03-07T03:20:12.217 INFO:tasks.workunit.client.0.smithi148.stdout:clone image286 -> image287 2024-03-07T03:20:12.301 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:12 smithi195 bash[66542]: cluster 2024-03-07T03:20:11.174800+0000 mon.a (mon.0) 449 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2024-03-07T03:20:12.464 INFO:tasks.workunit.client.0.smithi148.stdout:resize: 111705418 2024-03-07T03:20:12.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:12 smithi148 bash[79819]: cluster 2024-03-07T03:20:11.174800+0000 mon.a (mon.0) 449 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2024-03-07T03:20:12.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:12 smithi148 bash[82222]: cluster 2024-03-07T03:20:11.174800+0000 mon.a (mon.0) 449 : cluster 0 osdmap e434: 8 total, 8 up, 8 in 2024-03-07T03:20:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:12 smithi195 bash[59134]: ts=2024-03-07T03:20:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:13.006 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (4228 ms) 2024-03-07T03:20:13.006 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-03-07T03:20:13.062 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:621: Skipped 2024-03-07T03:20:13.062 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:13.195 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (189 ms) 2024-03-07T03:20:13.195 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-03-07T03:20:13.254 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:635: Skipped 2024-03-07T03:20:13.254 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:13.370 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (175 ms) 2024-03-07T03:20:13.370 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-03-07T03:20:13.420 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:649: Skipped 2024-03-07T03:20:13.420 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:13.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: cluster 2024-03-07T03:20:12.187101+0000 mon.a (mon.0) 450 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2024-03-07T03:20:13.496 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: cluster 2024-03-07T03:20:12.262500+0000 mgr.y (mgr.34101) 147 : cluster 0 pgmap v242: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:20:13.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: audit 2024-03-07T03:20:12.304859+0000 mon.a (mon.0) 451 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:13.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: audit 2024-03-07T03:20:12.525806+0000 mon.a (mon.0) 452 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: audit 2024-03-07T03:20:12.538075+0000 mon.a (mon.0) 453 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: audit 2024-03-07T03:20:12.587800+0000 mon.a (mon.0) 454 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:13.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: audit 2024-03-07T03:20:13.165747+0000 mon.a (mon.0) 455 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.497 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[66542]: audit 2024-03-07T03:20:13.172748+0000 mon.a (mon.0) 456 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: cluster 2024-03-07T03:20:12.187101+0000 mon.a (mon.0) 450 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: cluster 2024-03-07T03:20:12.262500+0000 mgr.y (mgr.34101) 147 : cluster 0 pgmap v242: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: audit 2024-03-07T03:20:12.304859+0000 mon.a (mon.0) 451 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: audit 2024-03-07T03:20:12.525806+0000 mon.a (mon.0) 452 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: audit 2024-03-07T03:20:12.538075+0000 mon.a (mon.0) 453 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: audit 2024-03-07T03:20:12.587800+0000 mon.a (mon.0) 454 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: audit 2024-03-07T03:20:13.165747+0000 mon.a (mon.0) 455 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[79819]: audit 2024-03-07T03:20:13.172748+0000 mon.a (mon.0) 456 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: cluster 2024-03-07T03:20:12.187101+0000 mon.a (mon.0) 450 : cluster 0 osdmap e435: 8 total, 8 up, 8 in 2024-03-07T03:20:13.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: cluster 2024-03-07T03:20:12.262500+0000 mgr.y (mgr.34101) 147 : cluster 0 pgmap v242: 105 pgs: 105 active+clean; 759 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:20:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: audit 2024-03-07T03:20:12.304859+0000 mon.a (mon.0) 451 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: audit 2024-03-07T03:20:12.525806+0000 mon.a (mon.0) 452 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: audit 2024-03-07T03:20:12.538075+0000 mon.a (mon.0) 453 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: audit 2024-03-07T03:20:12.587800+0000 mon.a (mon.0) 454 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: audit 2024-03-07T03:20:13.165747+0000 mon.a (mon.0) 455 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:13 smithi148 bash[82222]: audit 2024-03-07T03:20:13.172748+0000 mon.a (mon.0) 456 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:13.561 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingLargerDstObjSize (191 ms) 2024-03-07T03:20:13.561 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-03-07T03:20:13.610 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:663: Skipped 2024-03-07T03:20:13.610 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:13.730 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (169 ms) 2024-03-07T03:20:13.730 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-03-07T03:20:13.773 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:679: Skipped 2024-03-07T03:20:13.773 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:13.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:13 smithi195 bash[59134]: ts=2024-03-07T03:20:13.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:13.937 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingLargerDstObjSize (207 ms) 2024-03-07T03:20:13.937 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-03-07T03:20:13.999 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:693: Skipped 2024-03-07T03:20:14.000 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:14.173 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (236 ms) 2024-03-07T03:20:14.173 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-03-07T03:20:14.236 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:707: Skipped 2024-03-07T03:20:14.236 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:14.409 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (236 ms) 2024-03-07T03:20:14.409 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-03-07T03:20:14.469 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:721: Skipped 2024-03-07T03:20:14.469 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:14.643 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (234 ms) 2024-03-07T03:20:14.643 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-03-07T03:20:14.704 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:735: Skipped 2024-03-07T03:20:14.704 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:14.880 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (237 ms) 2024-03-07T03:20:14.880 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-03-07T03:20:14.940 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_DeepCopy.cc:751: Skipped 2024-03-07T03:20:14.940 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:15.114 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (234 ms) 2024-03-07T03:20:15.322 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 31 tests from TestDeepCopy (62055 ms total) 2024-03-07T03:20:15.322 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:20:15.323 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 51 tests from TestMirroring 2024-03-07T03:20:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:15 smithi148 bash[82222]: cluster 2024-03-07T03:20:14.263860+0000 mgr.y (mgr.34101) 148 : cluster 0 pgmap v243: 105 pgs: 105 active+clean; 951 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s rd, 36 MiB/s wr, 749 op/s 2024-03-07T03:20:15.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:15 smithi148 bash[79819]: cluster 2024-03-07T03:20:14.263860+0000 mgr.y (mgr.34101) 148 : cluster 0 pgmap v243: 105 pgs: 105 active+clean; 951 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s rd, 36 MiB/s wr, 749 op/s 2024-03-07T03:20:15.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:15 smithi195 bash[66542]: cluster 2024-03-07T03:20:14.263860+0000 mgr.y (mgr.34101) 148 : cluster 0 pgmap v243: 105 pgs: 105 active+clean; 951 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 45 MiB/s rd, 36 MiB/s wr, 749 op/s 2024-03-07T03:20:16.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:16 smithi148 bash[79819]: cluster 2024-03-07T03:20:15.329432+0000 mon.a (mon.0) 457 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2024-03-07T03:20:16.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:16 smithi148 bash[82222]: cluster 2024-03-07T03:20:15.329432+0000 mon.a (mon.0) 457 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2024-03-07T03:20:16.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:16 smithi195 bash[66542]: cluster 2024-03-07T03:20:15.329432+0000 mon.a (mon.0) 457 : cluster 0 osdmap e436: 8 total, 8 up, 8 in 2024-03-07T03:20:17.342 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-03-07T03:20:17.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:20:17] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2024-03-07T03:20:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[82222]: cluster 2024-03-07T03:20:16.264488+0000 mgr.y (mgr.34101) 149 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-07T03:20:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[82222]: cluster 2024-03-07T03:20:16.355384+0000 mon.a (mon.0) 458 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2024-03-07T03:20:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[82222]: audit 2024-03-07T03:20:16.365809+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[82222]: audit 2024-03-07T03:20:16.369475+0000 mon.a (mon.0) 459 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[79819]: cluster 2024-03-07T03:20:16.264488+0000 mgr.y (mgr.34101) 149 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-07T03:20:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[79819]: cluster 2024-03-07T03:20:16.355384+0000 mon.a (mon.0) 458 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2024-03-07T03:20:17.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[79819]: audit 2024-03-07T03:20:16.365809+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:17.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:17 smithi148 bash[79819]: audit 2024-03-07T03:20:16.369475+0000 mon.a (mon.0) 459 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:17 smithi195 bash[66542]: cluster 2024-03-07T03:20:16.264488+0000 mgr.y (mgr.34101) 149 : cluster 0 pgmap v245: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-03-07T03:20:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:17 smithi195 bash[66542]: cluster 2024-03-07T03:20:16.355384+0000 mon.a (mon.0) 458 : cluster 0 osdmap e437: 8 total, 8 up, 8 in 2024-03-07T03:20:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:17 smithi195 bash[66542]: audit 2024-03-07T03:20:16.365809+0000 mon.c (mon.2) 4 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:17 smithi195 bash[66542]: audit 2024-03-07T03:20:16.369475+0000 mon.a (mon.0) 459 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:20:18.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:18 smithi148 bash[79819]: audit 2024-03-07T03:20:17.347157+0000 mon.a (mon.0) 460 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:18 smithi148 bash[79819]: cluster 2024-03-07T03:20:17.353866+0000 mon.a (mon.0) 461 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2024-03-07T03:20:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:18 smithi148 bash[82222]: audit 2024-03-07T03:20:17.347157+0000 mon.a (mon.0) 460 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:18 smithi148 bash[82222]: cluster 2024-03-07T03:20:17.353866+0000 mon.a (mon.0) 461 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2024-03-07T03:20:18.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:18 smithi195 bash[66542]: audit 2024-03-07T03:20:17.347157+0000 mon.a (mon.0) 460 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:20:18.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:18 smithi195 bash[66542]: cluster 2024-03-07T03:20:17.353866+0000 mon.a (mon.0) 461 : cluster 0 osdmap e438: 8 total, 8 up, 8 in 2024-03-07T03:20:19.567 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2225 ms) 2024-03-07T03:20:19.567 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-03-07T03:20:19.749 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (182 ms) 2024-03-07T03:20:19.749 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-03-07T03:20:19.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[79819]: cluster 2024-03-07T03:20:18.265191+0000 mgr.y (mgr.34101) 150 : cluster 0 pgmap v248: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:19.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[79819]: cluster 2024-03-07T03:20:18.380855+0000 mon.a (mon.0) 462 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2024-03-07T03:20:19.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[79819]: audit 2024-03-07T03:20:19.006206+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:19.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[79819]: audit 2024-03-07T03:20:19.012553+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:19.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[82222]: cluster 2024-03-07T03:20:18.265191+0000 mgr.y (mgr.34101) 150 : cluster 0 pgmap v248: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:19.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[82222]: cluster 2024-03-07T03:20:18.380855+0000 mon.a (mon.0) 462 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2024-03-07T03:20:19.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[82222]: audit 2024-03-07T03:20:19.006206+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:19.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:19 smithi148 bash[82222]: audit 2024-03-07T03:20:19.012553+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:19 smithi195 bash[66542]: cluster 2024-03-07T03:20:18.265191+0000 mgr.y (mgr.34101) 150 : cluster 0 pgmap v248: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:19 smithi195 bash[66542]: cluster 2024-03-07T03:20:18.380855+0000 mon.a (mon.0) 462 : cluster 0 osdmap e439: 8 total, 8 up, 8 in 2024-03-07T03:20:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:19 smithi195 bash[66542]: audit 2024-03-07T03:20:19.006206+0000 mon.a (mon.0) 463 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:19 smithi195 bash[66542]: audit 2024-03-07T03:20:19.012553+0000 mon.a (mon.0) 464 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:19.867 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (118 ms) 2024-03-07T03:20:19.867 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-03-07T03:20:20.029 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (163 ms) 2024-03-07T03:20:20.030 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-03-07T03:20:20.189 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (159 ms) 2024-03-07T03:20:20.189 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-03-07T03:20:20.416 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (227 ms) 2024-03-07T03:20:20.416 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-03-07T03:20:20.684 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (268 ms) 2024-03-07T03:20:20.684 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-03-07T03:20:20.684 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_mirroring.cc:482: Skipped 2024-03-07T03:20:20.685 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:20.685 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-03-07T03:20:20.685 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-03-07T03:20:20.685 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_mirroring.cc:520: Skipped 2024-03-07T03:20:20.685 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:20:20.685 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-03-07T03:20:20.685 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[79819]: cluster 2024-03-07T03:20:19.383355+0000 mon.a (mon.0) 465 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[79819]: audit 2024-03-07T03:20:19.580139+0000 mon.a (mon.0) 466 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[79819]: audit 2024-03-07T03:20:19.580944+0000 mon.a (mon.0) 467 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[79819]: audit 2024-03-07T03:20:19.588072+0000 mon.a (mon.0) 468 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[82222]: cluster 2024-03-07T03:20:19.383355+0000 mon.a (mon.0) 465 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[82222]: audit 2024-03-07T03:20:19.580139+0000 mon.a (mon.0) 466 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[82222]: audit 2024-03-07T03:20:19.580944+0000 mon.a (mon.0) 467 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:20.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:20 smithi148 bash[82222]: audit 2024-03-07T03:20:19.588072+0000 mon.a (mon.0) 468 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:20 smithi195 bash[66542]: cluster 2024-03-07T03:20:19.383355+0000 mon.a (mon.0) 465 : cluster 0 osdmap e440: 8 total, 8 up, 8 in 2024-03-07T03:20:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:20 smithi195 bash[66542]: audit 2024-03-07T03:20:19.580139+0000 mon.a (mon.0) 466 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:20 smithi195 bash[66542]: audit 2024-03-07T03:20:19.580944+0000 mon.a (mon.0) 467 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:20 smithi195 bash[66542]: audit 2024-03-07T03:20:19.588072+0000 mon.a (mon.0) 468 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:20.863 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (179 ms) 2024-03-07T03:20:20.863 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-03-07T03:20:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (197 ms) 2024-03-07T03:20:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-03-07T03:20:21.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:21 smithi148 bash[79819]: cluster 2024-03-07T03:20:20.265609+0000 mgr.y (mgr.34101) 151 : cluster 0 pgmap v251: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:21.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:21 smithi148 bash[82222]: cluster 2024-03-07T03:20:20.265609+0000 mgr.y (mgr.34101) 151 : cluster 0 pgmap v251: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:21.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:21 smithi195 bash[66542]: cluster 2024-03-07T03:20:20.265609+0000 mgr.y (mgr.34101) 151 : cluster 0 pgmap v251: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:22.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:22 smithi195 bash[59134]: ts=2024-03-07T03:20:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:22.705 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1645 ms) 2024-03-07T03:20:22.705 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-03-07T03:20:23.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:22 smithi148 bash[79819]: cluster 2024-03-07T03:20:21.595516+0000 mon.a (mon.0) 469 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2024-03-07T03:20:23.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:22 smithi148 bash[79819]: cluster 2024-03-07T03:20:22.266163+0000 mgr.y (mgr.34101) 152 : cluster 0 pgmap v253: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:23.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:22 smithi148 bash[82222]: cluster 2024-03-07T03:20:21.595516+0000 mon.a (mon.0) 469 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2024-03-07T03:20:23.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:22 smithi148 bash[82222]: cluster 2024-03-07T03:20:22.266163+0000 mgr.y (mgr.34101) 152 : cluster 0 pgmap v253: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:23.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:22 smithi195 bash[66542]: cluster 2024-03-07T03:20:21.595516+0000 mon.a (mon.0) 469 : cluster 0 osdmap e441: 8 total, 8 up, 8 in 2024-03-07T03:20:23.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:22 smithi195 bash[66542]: cluster 2024-03-07T03:20:22.266163+0000 mgr.y (mgr.34101) 152 : cluster 0 pgmap v253: 105 pgs: 105 active+clean; 582 KiB data, 439 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:23.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:23 smithi195 bash[66542]: cluster 2024-03-07T03:20:22.606929+0000 mon.a (mon.0) 470 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2024-03-07T03:20:23.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:23 smithi195 bash[59134]: ts=2024-03-07T03:20:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:24.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:23 smithi148 bash[79819]: cluster 2024-03-07T03:20:22.606929+0000 mon.a (mon.0) 470 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2024-03-07T03:20:24.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:23 smithi148 bash[82222]: cluster 2024-03-07T03:20:22.606929+0000 mon.a (mon.0) 470 : cluster 0 osdmap e442: 8 total, 8 up, 8 in 2024-03-07T03:20:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (2008 ms) 2024-03-07T03:20:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-03-07T03:20:24.954 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (241 ms) 2024-03-07T03:20:24.954 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-03-07T03:20:25.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:24 smithi148 bash[79819]: cluster 2024-03-07T03:20:23.608441+0000 mon.a (mon.0) 471 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2024-03-07T03:20:25.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:24 smithi148 bash[79819]: cluster 2024-03-07T03:20:24.266850+0000 mgr.y (mgr.34101) 153 : cluster 0 pgmap v256: 105 pgs: 105 active+clean; 582 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 336 KiB/s rd, 45 KiB/s wr, 487 op/s 2024-03-07T03:20:25.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:24 smithi148 bash[82222]: cluster 2024-03-07T03:20:23.608441+0000 mon.a (mon.0) 471 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2024-03-07T03:20:25.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:24 smithi148 bash[82222]: cluster 2024-03-07T03:20:24.266850+0000 mgr.y (mgr.34101) 153 : cluster 0 pgmap v256: 105 pgs: 105 active+clean; 582 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 336 KiB/s rd, 45 KiB/s wr, 487 op/s 2024-03-07T03:20:25.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:24 smithi195 bash[66542]: cluster 2024-03-07T03:20:23.608441+0000 mon.a (mon.0) 471 : cluster 0 osdmap e443: 8 total, 8 up, 8 in 2024-03-07T03:20:25.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:24 smithi195 bash[66542]: cluster 2024-03-07T03:20:24.266850+0000 mgr.y (mgr.34101) 153 : cluster 0 pgmap v256: 105 pgs: 105 active+clean; 582 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 336 KiB/s rd, 45 KiB/s wr, 487 op/s 2024-03-07T03:20:25.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (248 ms) 2024-03-07T03:20:25.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-03-07T03:20:25.472 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (270 ms) 2024-03-07T03:20:25.472 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-03-07T03:20:25.677 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (205 ms) 2024-03-07T03:20:25.677 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-03-07T03:20:25.891 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (214 ms) 2024-03-07T03:20:25.891 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-03-07T03:20:26.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:25 smithi148 bash[79819]: cluster 2024-03-07T03:20:24.628275+0000 mon.a (mon.0) 472 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2024-03-07T03:20:26.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:25 smithi148 bash[82222]: cluster 2024-03-07T03:20:24.628275+0000 mon.a (mon.0) 472 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2024-03-07T03:20:26.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:25 smithi195 bash[66542]: cluster 2024-03-07T03:20:24.628275+0000 mon.a (mon.0) 472 : cluster 0 osdmap e444: 8 total, 8 up, 8 in 2024-03-07T03:20:26.152 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (261 ms) 2024-03-07T03:20:26.152 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-03-07T03:20:26.421 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (269 ms) 2024-03-07T03:20:26.421 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-03-07T03:20:27.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:26 smithi148 bash[79819]: cluster 2024-03-07T03:20:26.267512+0000 mgr.y (mgr.34101) 154 : cluster 0 pgmap v258: 105 pgs: 105 active+clean; 582 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 352 KiB/s rd, 47 KiB/s wr, 509 op/s 2024-03-07T03:20:27.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:26 smithi148 bash[82222]: cluster 2024-03-07T03:20:26.267512+0000 mgr.y (mgr.34101) 154 : cluster 0 pgmap v258: 105 pgs: 105 active+clean; 582 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 352 KiB/s rd, 47 KiB/s wr, 509 op/s 2024-03-07T03:20:27.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:26 smithi195 bash[66542]: cluster 2024-03-07T03:20:26.267512+0000 mgr.y (mgr.34101) 154 : cluster 0 pgmap v258: 105 pgs: 105 active+clean; 582 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 352 KiB/s rd, 47 KiB/s wr, 509 op/s 2024-03-07T03:20:27.195 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (774 ms) 2024-03-07T03:20:27.195 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-03-07T03:20:27.476 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (281 ms) 2024-03-07T03:20:27.476 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-03-07T03:20:27.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:20:27] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.43.0" 2024-03-07T03:20:27.700 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (225 ms) 2024-03-07T03:20:27.701 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-03-07T03:20:27.897 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (196 ms) 2024-03-07T03:20:27.897 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-03-07T03:20:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[79819]: cluster 2024-03-07T03:20:26.642602+0000 mon.a (mon.0) 473 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2024-03-07T03:20:28.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[79819]: cluster 2024-03-07T03:20:27.056920+0000 mon.a (mon.0) 474 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2024-03-07T03:20:28.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[79819]: audit 2024-03-07T03:20:27.305661+0000 mon.a (mon.0) 475 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:28.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[79819]: audit 2024-03-07T03:20:27.391055+0000 mon.a (mon.0) 476 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[82222]: cluster 2024-03-07T03:20:26.642602+0000 mon.a (mon.0) 473 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2024-03-07T03:20:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[82222]: cluster 2024-03-07T03:20:27.056920+0000 mon.a (mon.0) 474 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2024-03-07T03:20:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[82222]: audit 2024-03-07T03:20:27.305661+0000 mon.a (mon.0) 475 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:27 smithi148 bash[82222]: audit 2024-03-07T03:20:27.391055+0000 mon.a (mon.0) 476 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:28.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:27 smithi195 bash[66542]: cluster 2024-03-07T03:20:26.642602+0000 mon.a (mon.0) 473 : cluster 0 osdmap e445: 8 total, 8 up, 8 in 2024-03-07T03:20:28.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:27 smithi195 bash[66542]: cluster 2024-03-07T03:20:27.056920+0000 mon.a (mon.0) 474 : cluster 0 osdmap e446: 8 total, 8 up, 8 in 2024-03-07T03:20:28.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:27 smithi195 bash[66542]: audit 2024-03-07T03:20:27.305661+0000 mon.a (mon.0) 475 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:28.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:27 smithi195 bash[66542]: audit 2024-03-07T03:20:27.391055+0000 mon.a (mon.0) 476 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:28.260 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (363 ms) 2024-03-07T03:20:28.261 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-03-07T03:20:28.568 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (308 ms) 2024-03-07T03:20:28.568 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-03-07T03:20:28.619 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade resume' 2024-03-07T03:20:28.862 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (294 ms) 2024-03-07T03:20:28.862 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-03-07T03:20:29.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:28 smithi148 bash[79819]: cluster 2024-03-07T03:20:28.268956+0000 mgr.y (mgr.34101) 155 : cluster 0 pgmap v261: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 479 KiB/s rd, 61 KiB/s wr, 694 op/s 2024-03-07T03:20:29.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:28 smithi148 bash[82222]: cluster 2024-03-07T03:20:28.268956+0000 mgr.y (mgr.34101) 155 : cluster 0 pgmap v261: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 479 KiB/s rd, 61 KiB/s wr, 694 op/s 2024-03-07T03:20:29.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:28 smithi195 bash[66542]: cluster 2024-03-07T03:20:28.268956+0000 mgr.y (mgr.34101) 155 : cluster 0 pgmap v261: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 479 KiB/s rd, 61 KiB/s wr, 694 op/s 2024-03-07T03:20:29.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (340 ms) 2024-03-07T03:20:29.202 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-03-07T03:20:29.566 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (364 ms) 2024-03-07T03:20:29.566 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-03-07T03:20:29.956 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (390 ms) 2024-03-07T03:20:29.956 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-03-07T03:20:30.131 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (176 ms) 2024-03-07T03:20:30.132 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-03-07T03:20:30.283 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (151 ms) 2024-03-07T03:20:30.283 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-03-07T03:20:30.423 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (140 ms) 2024-03-07T03:20:30.423 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-03-07T03:20:30.656 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (233 ms) 2024-03-07T03:20:30.656 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-03-07T03:20:30.931 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (275 ms) 2024-03-07T03:20:30.931 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-03-07T03:20:31.172 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (242 ms) 2024-03-07T03:20:31.173 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-03-07T03:20:31.411 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (238 ms) 2024-03-07T03:20:31.411 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-03-07T03:20:31.552 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.MirrorStatusList (141 ms) 2024-03-07T03:20:31.552 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-03-07T03:20:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:31 smithi195 bash[66542]: cluster 2024-03-07T03:20:30.269803+0000 mgr.y (mgr.34101) 156 : cluster 0 pgmap v262: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 372 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-03-07T03:20:31.745 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.RemoveBootstrapped (194 ms) 2024-03-07T03:20:31.746 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-03-07T03:20:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:31 smithi148 bash[79819]: cluster 2024-03-07T03:20:30.269803+0000 mgr.y (mgr.34101) 156 : cluster 0 pgmap v262: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 372 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-03-07T03:20:31.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:31 smithi148 bash[82222]: cluster 2024-03-07T03:20:30.269803+0000 mgr.y (mgr.34101) 156 : cluster 0 pgmap v262: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 372 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-03-07T03:20:32.405 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.AioPromoteDemote (659 ms) 2024-03-07T03:20:32.405 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-03-07T03:20:32.626 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:32 smithi195 bash[59134]: ts=2024-03-07T03:20:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:33.038 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.AioGetInfo (633 ms) 2024-03-07T03:20:33.038 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-03-07T03:20:33.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:33 smithi195 bash[66542]: cluster 2024-03-07T03:20:32.056129+0000 mon.a (mon.0) 477 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2024-03-07T03:20:33.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:33 smithi195 bash[66542]: cluster 2024-03-07T03:20:32.270362+0000 mgr.y (mgr.34101) 157 : cluster 0 pgmap v264: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 372 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-03-07T03:20:33.335 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:20:33.354 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:33 smithi148 bash[82222]: cluster 2024-03-07T03:20:32.056129+0000 mon.a (mon.0) 477 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2024-03-07T03:20:33.354 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:33 smithi148 bash[82222]: cluster 2024-03-07T03:20:32.270362+0000 mgr.y (mgr.34101) 157 : cluster 0 pgmap v264: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 372 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-03-07T03:20:33.354 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:33 smithi148 bash[79819]: cluster 2024-03-07T03:20:32.056129+0000 mon.a (mon.0) 477 : cluster 0 osdmap e447: 8 total, 8 up, 8 in 2024-03-07T03:20:33.355 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:33 smithi148 bash[79819]: cluster 2024-03-07T03:20:32.270362+0000 mgr.y (mgr.34101) 157 : cluster 0 pgmap v264: 105 pgs: 105 active+clean; 582 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 372 KiB/s rd, 48 KiB/s wr, 539 op/s 2024-03-07T03:20:33.425 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.AioGetStatus (387 ms) 2024-03-07T03:20:33.425 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.SiteName 2024-03-07T03:20:33.435 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.SiteName (10 ms) 2024-03-07T03:20:33.435 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.Bootstrap 2024-03-07T03:20:33.812 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:33 smithi195 bash[59134]: ts=2024-03-07T03:20:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi148\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", domainname=\"(none)\", instance=\"smithi148\", job=\"node\", machine=\"x86_64\", nodename=\"smithi148\", release=\"5.15.0-97-generic\", sysname=\"Linux\", version=\"#107-Ubuntu SMP Wed Feb 7 13:26:48 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:34.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[79819]: audit 2024-03-07T03:20:33.443975+0000 mon.a (mon.0) 478 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[79819]: audit 2024-03-07T03:20:33.444665+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[79819]: audit 2024-03-07T03:20:33.444992+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[79819]: audit 2024-03-07T03:20:33.445261+0000 mon.a (mon.0) 479 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[79819]: audit 2024-03-07T03:20:33.448864+0000 mon.a (mon.0) 480 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[79819]: audit 2024-03-07T03:20:33.449454+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[82222]: audit 2024-03-07T03:20:33.443975+0000 mon.a (mon.0) 478 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[82222]: audit 2024-03-07T03:20:33.444665+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[82222]: audit 2024-03-07T03:20:33.444992+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[82222]: audit 2024-03-07T03:20:33.445261+0000 mon.a (mon.0) 479 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[82222]: audit 2024-03-07T03:20:33.448864+0000 mon.a (mon.0) 480 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-07T03:20:34.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:34 smithi148 bash[82222]: audit 2024-03-07T03:20:33.449454+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:34 smithi195 bash[66542]: audit 2024-03-07T03:20:33.443975+0000 mon.a (mon.0) 478 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:34.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:34 smithi195 bash[66542]: audit 2024-03-07T03:20:33.444665+0000 mon.c (mon.2) 5 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:34 smithi195 bash[66542]: audit 2024-03-07T03:20:33.444992+0000 mon.c (mon.2) 6 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:34 smithi195 bash[66542]: audit 2024-03-07T03:20:33.445261+0000 mon.a (mon.0) 479 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:34.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:34 smithi195 bash[66542]: audit 2024-03-07T03:20:33.448864+0000 mon.a (mon.0) 480 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-03-07T03:20:34.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:34 smithi195 bash[66542]: audit 2024-03-07T03:20:33.449454+0000 mon.c (mon.2) 7 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:35.134 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.Bootstrap (1699 ms) 2024-03-07T03:20:35.134 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.PeerDirection 2024-03-07T03:20:35.152 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.PeerDirection (18 ms) 2024-03-07T03:20:35.152 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.Snapshot 2024-03-07T03:20:35.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: cluster 2024-03-07T03:20:34.271205+0000 mgr.y (mgr.34101) 158 : cluster 0 pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 158 KiB/s wr, 1.43k op/s 2024-03-07T03:20:35.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.127457+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-07T03:20:35.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.132201+0000 mon.a (mon.0) 481 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:35.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.132849+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:20:35.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.133182+0000 mon.a (mon.0) 482 : audit 1 from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.135312+0000 mon.a (mon.0) 483 : audit 1 from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.137743+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.158171+0000 mon.c (mon.2) 11 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]: dispatch 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.158529+0000 mon.a (mon.0) 484 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]: dispatch 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[82222]: audit 2024-03-07T03:20:35.162210+0000 mon.a (mon.0) 485 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]': finished 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: cluster 2024-03-07T03:20:34.271205+0000 mgr.y (mgr.34101) 158 : cluster 0 pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 158 KiB/s wr, 1.43k op/s 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.127457+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-07T03:20:35.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.132201+0000 mon.a (mon.0) 481 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:35.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.132849+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:20:35.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.133182+0000 mon.a (mon.0) 482 : audit 1 from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:20:35.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.135312+0000 mon.a (mon.0) 483 : audit 1 from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-07T03:20:35.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.137743+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:35.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.158171+0000 mon.c (mon.2) 11 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]: dispatch 2024-03-07T03:20:35.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.158529+0000 mon.a (mon.0) 484 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]: dispatch 2024-03-07T03:20:35.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:35 smithi148 bash[79819]: audit 2024-03-07T03:20:35.162210+0000 mon.a (mon.0) 485 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]': finished 2024-03-07T03:20:35.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: cluster 2024-03-07T03:20:34.271205+0000 mgr.y (mgr.34101) 158 : cluster 0 pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 158 KiB/s wr, 1.43k op/s 2024-03-07T03:20:35.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.127457+0000 mon.c (mon.2) 8 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"}]: dispatch 2024-03-07T03:20:35.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.132201+0000 mon.a (mon.0) 481 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:20:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.132849+0000 mon.c (mon.2) 9 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:20:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.133182+0000 mon.a (mon.0) 482 : audit 1 from='client.? ' entity='client.admin' cmd=[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]: dispatch 2024-03-07T03:20:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.135312+0000 mon.a (mon.0) 483 : audit 1 from='client.? ' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-03-07T03:20:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.137743+0000 mon.c (mon.2) 10 : audit 0 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/site_name"}]: dispatch 2024-03-07T03:20:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.158171+0000 mon.c (mon.2) 11 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]: dispatch 2024-03-07T03:20:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.158529+0000 mon.a (mon.0) 484 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]: dispatch 2024-03-07T03:20:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:35 smithi195 bash[66542]: audit 2024-03-07T03:20:35.162210+0000 mon.a (mon.0) 485 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dde3a023-6f59-4f5e-b811-ace9720c5acc"}]': finished 2024-03-07T03:20:37.078 INFO:teuthology.orchestra.run.smithi148.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:20:37] "GET /metrics HTTP/1.1" 200 36374 "" "Prometheus/2.43.0" 2024-03-07T03:20:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[79819]: cluster 2024-03-07T03:20:36.169447+0000 mon.a (mon.0) 486 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2024-03-07T03:20:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[79819]: cluster 2024-03-07T03:20:36.271933+0000 mgr.y (mgr.34101) 159 : cluster 0 pgmap v267: 105 pgs: 105 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 712 KiB/s rd, 115 KiB/s wr, 964 op/s 2024-03-07T03:20:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[79819]: audit 2024-03-07T03:20:37.091762+0000 mon.a (mon.0) 487 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[79819]: audit 2024-03-07T03:20:37.096895+0000 mon.a (mon.0) 488 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[82222]: cluster 2024-03-07T03:20:36.169447+0000 mon.a (mon.0) 486 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2024-03-07T03:20:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[82222]: cluster 2024-03-07T03:20:36.271933+0000 mgr.y (mgr.34101) 159 : cluster 0 pgmap v267: 105 pgs: 105 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 712 KiB/s rd, 115 KiB/s wr, 964 op/s 2024-03-07T03:20:37.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[82222]: audit 2024-03-07T03:20:37.091762+0000 mon.a (mon.0) 487 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:37.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:37 smithi148 bash[82222]: audit 2024-03-07T03:20:37.096895+0000 mon.a (mon.0) 488 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:37 smithi195 bash[66542]: cluster 2024-03-07T03:20:36.169447+0000 mon.a (mon.0) 486 : cluster 0 osdmap e448: 8 total, 8 up, 8 in 2024-03-07T03:20:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:37 smithi195 bash[66542]: cluster 2024-03-07T03:20:36.271933+0000 mgr.y (mgr.34101) 159 : cluster 0 pgmap v267: 105 pgs: 105 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 712 KiB/s rd, 115 KiB/s wr, 964 op/s 2024-03-07T03:20:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:37 smithi195 bash[66542]: audit 2024-03-07T03:20:37.091762+0000 mon.a (mon.0) 487 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:37 smithi195 bash[66542]: audit 2024-03-07T03:20:37.096895+0000 mon.a (mon.0) 488 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:37.872 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for all mons to upgrade' 2024-03-07T03:20:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:38 smithi148 bash[79819]: audit 2024-03-07T03:20:37.086356+0000 mgr.y (mgr.34101) 160 : audit 0 from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:20:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:38 smithi148 bash[79819]: cephadm 2024-03-07T03:20:37.086764+0000 mgr.y (mgr.34101) 161 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:38 smithi148 bash[79819]: cluster 2024-03-07T03:20:37.172678+0000 mon.a (mon.0) 489 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2024-03-07T03:20:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:38 smithi148 bash[82222]: audit 2024-03-07T03:20:37.086356+0000 mgr.y (mgr.34101) 160 : audit 0 from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:20:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:38 smithi148 bash[82222]: cephadm 2024-03-07T03:20:37.086764+0000 mgr.y (mgr.34101) 161 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:38 smithi148 bash[82222]: cluster 2024-03-07T03:20:37.172678+0000 mon.a (mon.0) 489 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2024-03-07T03:20:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:38 smithi195 bash[66542]: audit 2024-03-07T03:20:37.086356+0000 mgr.y (mgr.34101) 160 : audit 0 from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:20:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:38 smithi195 bash[66542]: cephadm 2024-03-07T03:20:37.086764+0000 mgr.y (mgr.34101) 161 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:20:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:38 smithi195 bash[66542]: cluster 2024-03-07T03:20:37.172678+0000 mon.a (mon.0) 489 : cluster 0 osdmap e449: 8 total, 8 up, 8 in 2024-03-07T03:20:39.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:39 smithi148 bash[79819]: cluster 2024-03-07T03:20:38.184791+0000 mon.a (mon.0) 490 : cluster 0 osdmap e450: 8 total, 8 up, 8 in 2024-03-07T03:20:39.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:39 smithi148 bash[79819]: cluster 2024-03-07T03:20:38.273315+0000 mgr.y (mgr.34101) 162 : cluster 0 pgmap v270: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 189 KiB/s wr, 1.71k op/s 2024-03-07T03:20:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:39 smithi148 bash[82222]: cluster 2024-03-07T03:20:38.184791+0000 mon.a (mon.0) 490 : cluster 0 osdmap e450: 8 total, 8 up, 8 in 2024-03-07T03:20:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:39 smithi148 bash[82222]: cluster 2024-03-07T03:20:38.273315+0000 mgr.y (mgr.34101) 162 : cluster 0 pgmap v270: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 189 KiB/s wr, 1.71k op/s 2024-03-07T03:20:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:39 smithi195 bash[66542]: cluster 2024-03-07T03:20:38.184791+0000 mon.a (mon.0) 490 : cluster 0 osdmap e450: 8 total, 8 up, 8 in 2024-03-07T03:20:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:39 smithi195 bash[66542]: cluster 2024-03-07T03:20:38.273315+0000 mgr.y (mgr.34101) 162 : cluster 0 pgmap v270: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 189 KiB/s wr, 1.71k op/s 2024-03-07T03:20:40.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:40 smithi148 bash[79819]: cluster 2024-03-07T03:20:39.194828+0000 mon.a (mon.0) 491 : cluster 0 osdmap e451: 8 total, 8 up, 8 in 2024-03-07T03:20:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:40 smithi148 bash[82222]: cluster 2024-03-07T03:20:39.194828+0000 mon.a (mon.0) 491 : cluster 0 osdmap e451: 8 total, 8 up, 8 in 2024-03-07T03:20:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:40 smithi195 bash[66542]: cluster 2024-03-07T03:20:39.194828+0000 mon.a (mon.0) 491 : cluster 0 osdmap e451: 8 total, 8 up, 8 in 2024-03-07T03:20:41.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:41 smithi148 bash[79819]: cluster 2024-03-07T03:20:40.197345+0000 mon.a (mon.0) 492 : cluster 0 osdmap e452: 8 total, 8 up, 8 in 2024-03-07T03:20:41.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:41 smithi148 bash[79819]: cluster 2024-03-07T03:20:40.274030+0000 mgr.y (mgr.34101) 163 : cluster 0 pgmap v273: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 470 KiB/s rd, 54 KiB/s wr, 643 op/s 2024-03-07T03:20:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:41 smithi148 bash[82222]: cluster 2024-03-07T03:20:40.197345+0000 mon.a (mon.0) 492 : cluster 0 osdmap e452: 8 total, 8 up, 8 in 2024-03-07T03:20:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:41 smithi148 bash[82222]: cluster 2024-03-07T03:20:40.274030+0000 mgr.y (mgr.34101) 163 : cluster 0 pgmap v273: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 470 KiB/s rd, 54 KiB/s wr, 643 op/s 2024-03-07T03:20:41.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:41 smithi195 bash[66542]: cluster 2024-03-07T03:20:40.197345+0000 mon.a (mon.0) 492 : cluster 0 osdmap e452: 8 total, 8 up, 8 in 2024-03-07T03:20:41.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:41 smithi195 bash[66542]: cluster 2024-03-07T03:20:40.274030+0000 mgr.y (mgr.34101) 163 : cluster 0 pgmap v273: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 470 KiB/s rd, 54 KiB/s wr, 643 op/s 2024-03-07T03:20:42.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:42 smithi148 bash[79819]: cluster 2024-03-07T03:20:41.223815+0000 mon.a (mon.0) 493 : cluster 0 osdmap e453: 8 total, 8 up, 8 in 2024-03-07T03:20:42.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:42 smithi148 bash[82222]: cluster 2024-03-07T03:20:41.223815+0000 mon.a (mon.0) 493 : cluster 0 osdmap e453: 8 total, 8 up, 8 in 2024-03-07T03:20:42.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:42 smithi195 bash[66542]: cluster 2024-03-07T03:20:41.223815+0000 mon.a (mon.0) 493 : cluster 0 osdmap e453: 8 total, 8 up, 8 in 2024-03-07T03:20:42.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:42 smithi195 bash[59134]: ts=2024-03-07T03:20:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:42.618 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:20:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[79819]: cluster 2024-03-07T03:20:42.226038+0000 mon.a (mon.0) 494 : cluster 0 osdmap e454: 8 total, 8 up, 8 in 2024-03-07T03:20:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[79819]: cluster 2024-03-07T03:20:42.274682+0000 mgr.y (mgr.34101) 164 : cluster 0 pgmap v276: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[79819]: audit 2024-03-07T03:20:42.305688+0000 mon.a (mon.0) 495 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[79819]: audit 2024-03-07T03:20:43.105230+0000 mon.a (mon.0) 496 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[79819]: audit 2024-03-07T03:20:43.117058+0000 mon.a (mon.0) 497 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[82222]: cluster 2024-03-07T03:20:42.226038+0000 mon.a (mon.0) 494 : cluster 0 osdmap e454: 8 total, 8 up, 8 in 2024-03-07T03:20:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[82222]: cluster 2024-03-07T03:20:42.274682+0000 mgr.y (mgr.34101) 164 : cluster 0 pgmap v276: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[82222]: audit 2024-03-07T03:20:42.305688+0000 mon.a (mon.0) 495 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[82222]: audit 2024-03-07T03:20:43.105230+0000 mon.a (mon.0) 496 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:43 smithi148 bash[82222]: audit 2024-03-07T03:20:43.117058+0000 mon.a (mon.0) 497 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:43 smithi195 bash[66542]: cluster 2024-03-07T03:20:42.226038+0000 mon.a (mon.0) 494 : cluster 0 osdmap e454: 8 total, 8 up, 8 in 2024-03-07T03:20:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:43 smithi195 bash[66542]: cluster 2024-03-07T03:20:42.274682+0000 mgr.y (mgr.34101) 164 : cluster 0 pgmap v276: 105 pgs: 105 active+clean; 583 KiB data, 429 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:43 smithi195 bash[66542]: audit 2024-03-07T03:20:42.305688+0000 mon.a (mon.0) 495 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:43 smithi195 bash[66542]: audit 2024-03-07T03:20:43.105230+0000 mon.a (mon.0) 496 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:43 smithi195 bash[66542]: audit 2024-03-07T03:20:43.117058+0000 mon.a (mon.0) 497 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: cluster 2024-03-07T03:20:43.247330+0000 mon.a (mon.0) 498 : cluster 0 osdmap e455: 8 total, 8 up, 8 in 2024-03-07T03:20:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.741479+0000 mon.a (mon.0) 499 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.742532+0000 mon.a (mon.0) 500 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.748763+0000 mon.a (mon.0) 501 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.807974+0000 mon.a (mon.0) 502 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.810108+0000 mon.a (mon.0) 503 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.812181+0000 mon.a (mon.0) 504 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.819285+0000 mon.a (mon.0) 505 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.821579+0000 mon.a (mon.0) 506 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.826879+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.828747+0000 mon.a (mon.0) 508 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.833723+0000 mon.a (mon.0) 509 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.835569+0000 mon.a (mon.0) 510 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.840232+0000 mon.a (mon.0) 511 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.842329+0000 mon.a (mon.0) 512 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.848674+0000 mon.a (mon.0) 513 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[79819]: audit 2024-03-07T03:20:43.851235+0000 mon.a (mon.0) 514 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: cluster 2024-03-07T03:20:43.247330+0000 mon.a (mon.0) 498 : cluster 0 osdmap e455: 8 total, 8 up, 8 in 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.741479+0000 mon.a (mon.0) 499 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.742532+0000 mon.a (mon.0) 500 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.748763+0000 mon.a (mon.0) 501 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.807974+0000 mon.a (mon.0) 502 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.810108+0000 mon.a (mon.0) 503 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.812181+0000 mon.a (mon.0) 504 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.819285+0000 mon.a (mon.0) 505 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.821579+0000 mon.a (mon.0) 506 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.826879+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.828747+0000 mon.a (mon.0) 508 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.833723+0000 mon.a (mon.0) 509 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-07T03:20:44.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.835569+0000 mon.a (mon.0) 510 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-07T03:20:44.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.840232+0000 mon.a (mon.0) 511 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-07T03:20:44.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.842329+0000 mon.a (mon.0) 512 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.848674+0000 mon.a (mon.0) 513 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:44 smithi148 bash[82222]: audit 2024-03-07T03:20:43.851235+0000 mon.a (mon.0) 514 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:20:44.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: cluster 2024-03-07T03:20:43.247330+0000 mon.a (mon.0) 498 : cluster 0 osdmap e455: 8 total, 8 up, 8 in 2024-03-07T03:20:44.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.741479+0000 mon.a (mon.0) 499 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.742532+0000 mon.a (mon.0) 500 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.748763+0000 mon.a (mon.0) 501 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.807974+0000 mon.a (mon.0) 502 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.810108+0000 mon.a (mon.0) 503 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.812181+0000 mon.a (mon.0) 504 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.819285+0000 mon.a (mon.0) 505 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.821579+0000 mon.a (mon.0) 506 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.826879+0000 mon.a (mon.0) 507 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.828747+0000 mon.a (mon.0) 508 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.833723+0000 mon.a (mon.0) 509 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.835569+0000 mon.a (mon.0) 510 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.840232+0000 mon.a (mon.0) 511 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.842329+0000 mon.a (mon.0) 512 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.848674+0000 mon.a (mon.0) 513 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:44.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:44 smithi195 bash[66542]: audit 2024-03-07T03:20:43.851235+0000 mon.a (mon.0) 514 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:20:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[79819]: cephadm 2024-03-07T03:20:43.813276+0000 mgr.y (mgr.34101) 165 : cephadm 1 Upgrade: Setting container_image for all mon 2024-03-07T03:20:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[79819]: cephadm 2024-03-07T03:20:43.843134+0000 mgr.y (mgr.34101) 166 : cephadm 1 Upgrade: Setting container_image for all crash 2024-03-07T03:20:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[79819]: audit 2024-03-07T03:20:43.851404+0000 mgr.y (mgr.34101) 167 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:20:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[79819]: cephadm 2024-03-07T03:20:43.852317+0000 mgr.y (mgr.34101) 168 : cephadm 1 Upgrade: osd.0 is safe to restart 2024-03-07T03:20:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[79819]: cluster 2024-03-07T03:20:44.255026+0000 mon.a (mon.0) 515 : cluster 0 osdmap e456: 8 total, 8 up, 8 in 2024-03-07T03:20:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[79819]: cluster 2024-03-07T03:20:44.275062+0000 mgr.y (mgr.34101) 169 : cluster 0 pgmap v279: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:20:45.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[82222]: cephadm 2024-03-07T03:20:43.813276+0000 mgr.y (mgr.34101) 165 : cephadm 1 Upgrade: Setting container_image for all mon 2024-03-07T03:20:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[82222]: cephadm 2024-03-07T03:20:43.843134+0000 mgr.y (mgr.34101) 166 : cephadm 1 Upgrade: Setting container_image for all crash 2024-03-07T03:20:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[82222]: audit 2024-03-07T03:20:43.851404+0000 mgr.y (mgr.34101) 167 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:20:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[82222]: cephadm 2024-03-07T03:20:43.852317+0000 mgr.y (mgr.34101) 168 : cephadm 1 Upgrade: osd.0 is safe to restart 2024-03-07T03:20:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[82222]: cluster 2024-03-07T03:20:44.255026+0000 mon.a (mon.0) 515 : cluster 0 osdmap e456: 8 total, 8 up, 8 in 2024-03-07T03:20:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:45 smithi148 bash[82222]: cluster 2024-03-07T03:20:44.275062+0000 mgr.y (mgr.34101) 169 : cluster 0 pgmap v279: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:20:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:45 smithi195 bash[66542]: cephadm 2024-03-07T03:20:43.813276+0000 mgr.y (mgr.34101) 165 : cephadm 1 Upgrade: Setting container_image for all mon 2024-03-07T03:20:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:45 smithi195 bash[66542]: cephadm 2024-03-07T03:20:43.843134+0000 mgr.y (mgr.34101) 166 : cephadm 1 Upgrade: Setting container_image for all crash 2024-03-07T03:20:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:45 smithi195 bash[66542]: audit 2024-03-07T03:20:43.851404+0000 mgr.y (mgr.34101) 167 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-07T03:20:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:45 smithi195 bash[66542]: cephadm 2024-03-07T03:20:43.852317+0000 mgr.y (mgr.34101) 168 : cephadm 1 Upgrade: osd.0 is safe to restart 2024-03-07T03:20:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:45 smithi195 bash[66542]: cluster 2024-03-07T03:20:44.255026+0000 mon.a (mon.0) 515 : cluster 0 osdmap e456: 8 total, 8 up, 8 in 2024-03-07T03:20:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:45 smithi195 bash[66542]: cluster 2024-03-07T03:20:44.275062+0000 mgr.y (mgr.34101) 169 : cluster 0 pgmap v279: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:20:46.210 INFO:teuthology.orchestra.run.smithi148.stdout:wait for all mons to upgrade 2024-03-07T03:20:46.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:46 smithi148 bash[79819]: cluster 2024-03-07T03:20:45.266756+0000 mon.a (mon.0) 516 : cluster 0 osdmap e457: 8 total, 8 up, 8 in 2024-03-07T03:20:46.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:46 smithi148 bash[82222]: cluster 2024-03-07T03:20:45.266756+0000 mon.a (mon.0) 516 : cluster 0 osdmap e457: 8 total, 8 up, 8 in 2024-03-07T03:20:46.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:46 smithi195 bash[66542]: cluster 2024-03-07T03:20:45.266756+0000 mon.a (mon.0) 516 : cluster 0 osdmap e457: 8 total, 8 up, 8 in 2024-03-07T03:20:47.263 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:20:47] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2024-03-07T03:20:47.413 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-03-07T03:20:47.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:47 smithi148 bash[79819]: cluster 2024-03-07T03:20:46.269854+0000 mon.a (mon.0) 517 : cluster 0 osdmap e458: 8 total, 8 up, 8 in 2024-03-07T03:20:47.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:47 smithi148 bash[79819]: cluster 2024-03-07T03:20:46.275517+0000 mgr.y (mgr.34101) 170 : cluster 0 pgmap v282: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:20:47.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:47 smithi148 bash[79819]: cluster 2024-03-07T03:20:47.064086+0000 mon.a (mon.0) 518 : cluster 0 osdmap e459: 8 total, 8 up, 8 in 2024-03-07T03:20:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:47 smithi148 bash[82222]: cluster 2024-03-07T03:20:46.269854+0000 mon.a (mon.0) 517 : cluster 0 osdmap e458: 8 total, 8 up, 8 in 2024-03-07T03:20:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:47 smithi148 bash[82222]: cluster 2024-03-07T03:20:46.275517+0000 mgr.y (mgr.34101) 170 : cluster 0 pgmap v282: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:20:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:47 smithi148 bash[82222]: cluster 2024-03-07T03:20:47.064086+0000 mon.a (mon.0) 518 : cluster 0 osdmap e459: 8 total, 8 up, 8 in 2024-03-07T03:20:47.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:47 smithi195 bash[66542]: cluster 2024-03-07T03:20:46.269854+0000 mon.a (mon.0) 517 : cluster 0 osdmap e458: 8 total, 8 up, 8 in 2024-03-07T03:20:47.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:47 smithi195 bash[66542]: cluster 2024-03-07T03:20:46.275517+0000 mgr.y (mgr.34101) 170 : cluster 0 pgmap v282: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.0 KiB/s wr, 62 op/s 2024-03-07T03:20:47.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:47 smithi195 bash[66542]: cluster 2024-03-07T03:20:47.064086+0000 mon.a (mon.0) 518 : cluster 0 osdmap e459: 8 total, 8 up, 8 in 2024-03-07T03:20:48.736 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[82222]: cephadm 2024-03-07T03:20:47.424316+0000 mgr.y (mgr.34101) 171 : cephadm 1 Upgrade: Updating osd.0 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[82222]: audit 2024-03-07T03:20:47.433344+0000 mon.a (mon.0) 519 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[82222]: audit 2024-03-07T03:20:47.437921+0000 mon.a (mon.0) 520 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[82222]: audit 2024-03-07T03:20:47.438964+0000 mon.a (mon.0) 521 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[82222]: cephadm 2024-03-07T03:20:47.442066+0000 mgr.y (mgr.34101) 172 : cephadm 1 Deploying daemon osd.0 on smithi148 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[82222]: cluster 2024-03-07T03:20:48.080612+0000 mon.a (mon.0) 522 : cluster 0 osdmap e460: 8 total, 8 up, 8 in 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[79819]: cephadm 2024-03-07T03:20:47.424316+0000 mgr.y (mgr.34101) 171 : cephadm 1 Upgrade: Updating osd.0 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[79819]: audit 2024-03-07T03:20:47.433344+0000 mon.a (mon.0) 519 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[79819]: audit 2024-03-07T03:20:47.437921+0000 mon.a (mon.0) 520 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[79819]: audit 2024-03-07T03:20:47.438964+0000 mon.a (mon.0) 521 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[79819]: cephadm 2024-03-07T03:20:47.442066+0000 mgr.y (mgr.34101) 172 : cephadm 1 Deploying daemon osd.0 on smithi148 2024-03-07T03:20:48.737 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:48 smithi148 bash[79819]: cluster 2024-03-07T03:20:48.080612+0000 mon.a (mon.0) 522 : cluster 0 osdmap e460: 8 total, 8 up, 8 in 2024-03-07T03:20:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:48 smithi195 bash[66542]: cephadm 2024-03-07T03:20:47.424316+0000 mgr.y (mgr.34101) 171 : cephadm 1 Upgrade: Updating osd.0 2024-03-07T03:20:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:48 smithi195 bash[66542]: audit 2024-03-07T03:20:47.433344+0000 mon.a (mon.0) 519 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:48 smithi195 bash[66542]: audit 2024-03-07T03:20:47.437921+0000 mon.a (mon.0) 520 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-07T03:20:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:48 smithi195 bash[66542]: audit 2024-03-07T03:20:47.438964+0000 mon.a (mon.0) 521 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:20:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:48 smithi195 bash[66542]: cephadm 2024-03-07T03:20:47.442066+0000 mgr.y (mgr.34101) 172 : cephadm 1 Deploying daemon osd.0 on smithi148 2024-03-07T03:20:48.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:48 smithi195 bash[66542]: cluster 2024-03-07T03:20:48.080612+0000 mon.a (mon.0) 522 : cluster 0 osdmap e460: 8 total, 8 up, 8 in 2024-03-07T03:20:49.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:49 smithi148 bash[79819]: cluster 2024-03-07T03:20:48.276826+0000 mgr.y (mgr.34101) 173 : cluster 0 pgmap v285: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 7.2 KiB/s wr, 111 op/s 2024-03-07T03:20:49.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:49 smithi148 bash[79819]: cluster 2024-03-07T03:20:49.087105+0000 mon.a (mon.0) 523 : cluster 0 osdmap e461: 8 total, 8 up, 8 in 2024-03-07T03:20:49.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:49 smithi148 bash[82222]: cluster 2024-03-07T03:20:48.276826+0000 mgr.y (mgr.34101) 173 : cluster 0 pgmap v285: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 7.2 KiB/s wr, 111 op/s 2024-03-07T03:20:49.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:49 smithi148 bash[82222]: cluster 2024-03-07T03:20:49.087105+0000 mon.a (mon.0) 523 : cluster 0 osdmap e461: 8 total, 8 up, 8 in 2024-03-07T03:20:49.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:49 smithi195 bash[66542]: cluster 2024-03-07T03:20:48.276826+0000 mgr.y (mgr.34101) 173 : cluster 0 pgmap v285: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 7.2 KiB/s wr, 111 op/s 2024-03-07T03:20:49.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:49 smithi195 bash[66542]: cluster 2024-03-07T03:20:49.087105+0000 mon.a (mon.0) 523 : cluster 0 osdmap e461: 8 total, 8 up, 8 in 2024-03-07T03:20:51.281 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.Snapshot (16128 ms) 2024-03-07T03:20:51.281 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-03-07T03:20:51.505 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:51 smithi148 bash[79819]: cluster 2024-03-07T03:20:50.091607+0000 mon.a (mon.0) 524 : cluster 0 osdmap e462: 8 total, 8 up, 8 in 2024-03-07T03:20:51.506 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:51 smithi148 bash[79819]: cluster 2024-03-07T03:20:50.277473+0000 mgr.y (mgr.34101) 174 : cluster 0 pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 7.2 KiB/s wr, 111 op/s 2024-03-07T03:20:51.506 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:51 smithi148 bash[82222]: cluster 2024-03-07T03:20:50.091607+0000 mon.a (mon.0) 524 : cluster 0 osdmap e462: 8 total, 8 up, 8 in 2024-03-07T03:20:51.506 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:51 smithi148 bash[82222]: cluster 2024-03-07T03:20:50.277473+0000 mgr.y (mgr.34101) 174 : cluster 0 pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 7.2 KiB/s wr, 111 op/s 2024-03-07T03:20:51.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:51 smithi195 bash[66542]: cluster 2024-03-07T03:20:50.091607+0000 mon.a (mon.0) 524 : cluster 0 osdmap e462: 8 total, 8 up, 8 in 2024-03-07T03:20:51.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:51 smithi195 bash[66542]: cluster 2024-03-07T03:20:50.277473+0000 mgr.y (mgr.34101) 174 : cluster 0 pgmap v288: 105 pgs: 105 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 7.2 KiB/s wr, 111 op/s 2024-03-07T03:20:51.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.790 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.790 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.790 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.790 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.791 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.791 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: Stopping Ceph osd.0 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:20:51.791 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:51.791 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:20:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:52.119 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:20:52.140 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:51 smithi148 bash[34197]: debug 2024-03-07T03:20:51.858+0000 7fb4b2dbf700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:20:52.140 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:51 smithi148 bash[34197]: debug 2024-03-07T03:20:51.858+0000 7fb4b2dbf700 -1 osd.0 463 *** Got signal Terminated *** 2024-03-07T03:20:52.140 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:51 smithi148 bash[34197]: debug 2024-03-07T03:20:51.858+0000 7fb4b2dbf700 -1 osd.0 463 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:20:52.140 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: debug 2024-03-07T03:20:52.054+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:20:52.140 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: cluster 2024-03-07T03:20:51.102573+0000 mon.a (mon.0) 525 : cluster 0 osdmap e463: 8 total, 8 up, 8 in 2024-03-07T03:20:52.140 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: audit 2024-03-07T03:20:51.277260+0000 mon.c (mon.2) 12 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]: dispatch 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: audit 2024-03-07T03:20:51.278055+0000 mon.a (mon.0) 526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]: dispatch 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: audit 2024-03-07T03:20:51.284316+0000 mon.a (mon.0) 527 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]': finished 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: cluster 2024-03-07T03:20:51.862727+0000 mon.a (mon.0) 528 : cluster 1 osd.0 marked itself down and dead 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: cluster 2024-03-07T03:20:52.059385+0000 mon.a (mon.0) 529 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: cluster 2024-03-07T03:20:52.059440+0000 mon.a (mon.0) 530 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[79819]: cluster 2024-03-07T03:20:52.070256+0000 mon.a (mon.0) 531 : cluster 0 osdmap e464: 8 total, 7 up, 8 in 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: cluster 2024-03-07T03:20:51.102573+0000 mon.a (mon.0) 525 : cluster 0 osdmap e463: 8 total, 8 up, 8 in 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: audit 2024-03-07T03:20:51.277260+0000 mon.c (mon.2) 12 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]: dispatch 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: audit 2024-03-07T03:20:51.278055+0000 mon.a (mon.0) 526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]: dispatch 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: audit 2024-03-07T03:20:51.284316+0000 mon.a (mon.0) 527 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]': finished 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: cluster 2024-03-07T03:20:51.862727+0000 mon.a (mon.0) 528 : cluster 1 osd.0 marked itself down and dead 2024-03-07T03:20:52.141 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: cluster 2024-03-07T03:20:52.059385+0000 mon.a (mon.0) 529 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:20:52.142 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: cluster 2024-03-07T03:20:52.059440+0000 mon.a (mon.0) 530 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:20:52.142 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:52 smithi148 bash[82222]: cluster 2024-03-07T03:20:52.070256+0000 mon.a (mon.0) 531 : cluster 0 osdmap e464: 8 total, 7 up, 8 in 2024-03-07T03:20:52.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[59134]: ts=2024-03-07T03:20:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:20:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: cluster 2024-03-07T03:20:51.102573+0000 mon.a (mon.0) 525 : cluster 0 osdmap e463: 8 total, 8 up, 8 in 2024-03-07T03:20:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: audit 2024-03-07T03:20:51.277260+0000 mon.c (mon.2) 12 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]: dispatch 2024-03-07T03:20:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: audit 2024-03-07T03:20:51.278055+0000 mon.a (mon.0) 526 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]: dispatch 2024-03-07T03:20:52.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: audit 2024-03-07T03:20:51.284316+0000 mon.a (mon.0) 527 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e7aed815-2f52-4a95-ae35-95e6636f01ef"}]': finished 2024-03-07T03:20:52.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: cluster 2024-03-07T03:20:51.862727+0000 mon.a (mon.0) 528 : cluster 1 osd.0 marked itself down and dead 2024-03-07T03:20:52.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: cluster 2024-03-07T03:20:52.059385+0000 mon.a (mon.0) 529 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:20:52.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: cluster 2024-03-07T03:20:52.059440+0000 mon.a (mon.0) 530 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:20:52.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:52 smithi195 bash[66542]: cluster 2024-03-07T03:20:52.070256+0000 mon.a (mon.0) 531 : cluster 0 osdmap e464: 8 total, 7 up, 8 in 2024-03-07T03:20:53.450 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:53 smithi148 bash[79819]: cluster 2024-03-07T03:20:52.278177+0000 mgr.y (mgr.34101) 175 : cluster 0 pgmap v291: 105 pgs: 18 stale+active+clean, 87 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:53.450 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:53 smithi148 bash[79819]: cluster 2024-03-07T03:20:53.072346+0000 mon.a (mon.0) 532 : cluster 0 osdmap e465: 8 total, 7 up, 8 in 2024-03-07T03:20:53.450 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:53 smithi148 bash[82222]: cluster 2024-03-07T03:20:52.278177+0000 mgr.y (mgr.34101) 175 : cluster 0 pgmap v291: 105 pgs: 18 stale+active+clean, 87 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:53.450 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:53 smithi148 bash[82222]: cluster 2024-03-07T03:20:53.072346+0000 mon.a (mon.0) 532 : cluster 0 osdmap e465: 8 total, 7 up, 8 in 2024-03-07T03:20:53.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:53 smithi195 bash[66542]: cluster 2024-03-07T03:20:52.278177+0000 mgr.y (mgr.34101) 175 : cluster 0 pgmap v291: 105 pgs: 18 stale+active+clean, 87 active+clean; 583 KiB data, 414 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:20:53.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:53 smithi195 bash[66542]: cluster 2024-03-07T03:20:53.072346+0000 mon.a (mon.0) 532 : cluster 0 osdmap e465: 8 total, 7 up, 8 in 2024-03-07T03:20:53.742 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:53 smithi148 bash[88299]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:20:54.039 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:53 smithi148 bash[88375]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:20:55.467 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:55 smithi148 bash[79819]: cluster 2024-03-07T03:20:54.123582+0000 mon.a (mon.0) 533 : cluster 0 osdmap e466: 8 total, 7 up, 8 in 2024-03-07T03:20:55.467 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:55 smithi148 bash[79819]: cluster 2024-03-07T03:20:54.279242+0000 mgr.y (mgr.34101) 176 : cluster 0 pgmap v294: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 7.0 KiB/s wr, 89 op/s; 47/309 objects degraded (15.210%) 2024-03-07T03:20:55.468 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:55 smithi148 bash[82222]: cluster 2024-03-07T03:20:54.123582+0000 mon.a (mon.0) 533 : cluster 0 osdmap e466: 8 total, 7 up, 8 in 2024-03-07T03:20:55.468 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:55 smithi148 bash[82222]: cluster 2024-03-07T03:20:54.279242+0000 mgr.y (mgr.34101) 176 : cluster 0 pgmap v294: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 7.0 KiB/s wr, 89 op/s; 47/309 objects degraded (15.210%) 2024-03-07T03:20:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:55 smithi195 bash[66542]: cluster 2024-03-07T03:20:54.123582+0000 mon.a (mon.0) 533 : cluster 0 osdmap e466: 8 total, 7 up, 8 in 2024-03-07T03:20:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:55 smithi195 bash[66542]: cluster 2024-03-07T03:20:54.279242+0000 mgr.y (mgr.34101) 176 : cluster 0 pgmap v294: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 7.0 KiB/s wr, 89 op/s; 47/309 objects degraded (15.210%) 2024-03-07T03:20:56.304 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (5023 ms) 2024-03-07T03:20:56.304 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-03-07T03:20:56.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:56 smithi148 bash[79819]: cluster 2024-03-07T03:20:55.113663+0000 mon.a (mon.0) 534 : cluster 3 Health check failed: Degraded data redundancy: 47/309 objects degraded (15.210%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:20:56.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:56 smithi148 bash[79819]: cluster 2024-03-07T03:20:55.129980+0000 mon.a (mon.0) 535 : cluster 0 osdmap e467: 8 total, 7 up, 8 in 2024-03-07T03:20:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:56 smithi148 bash[82222]: cluster 2024-03-07T03:20:55.113663+0000 mon.a (mon.0) 534 : cluster 3 Health check failed: Degraded data redundancy: 47/309 objects degraded (15.210%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:20:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:56 smithi148 bash[82222]: cluster 2024-03-07T03:20:55.129980+0000 mon.a (mon.0) 535 : cluster 0 osdmap e467: 8 total, 7 up, 8 in 2024-03-07T03:20:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:56 smithi195 bash[66542]: cluster 2024-03-07T03:20:55.113663+0000 mon.a (mon.0) 534 : cluster 3 Health check failed: Degraded data redundancy: 47/309 objects degraded (15.210%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:20:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:56 smithi195 bash[66542]: cluster 2024-03-07T03:20:55.129980+0000 mon.a (mon.0) 535 : cluster 0 osdmap e467: 8 total, 7 up, 8 in 2024-03-07T03:20:57.165 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.165 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.165 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.165 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.166 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.166 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[79819]: cluster 2024-03-07T03:20:56.136226+0000 mon.a (mon.0) 536 : cluster 0 osdmap e468: 8 total, 7 up, 8 in 2024-03-07T03:20:57.166 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[79819]: cluster 2024-03-07T03:20:56.280483+0000 mgr.y (mgr.34101) 177 : cluster 0 pgmap v297: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 7.0 KiB/s wr, 89 op/s; 47/309 objects degraded (15.210%) 2024-03-07T03:20:57.166 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[79819]: audit 2024-03-07T03:20:56.293840+0000 mon.c (mon.2) 13 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]: dispatch 2024-03-07T03:20:57.166 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[79819]: audit 2024-03-07T03:20:56.294518+0000 mon.a (mon.0) 537 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]: dispatch 2024-03-07T03:20:57.166 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[79819]: audit 2024-03-07T03:20:56.302575+0000 mon.a (mon.0) 538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]': finished 2024-03-07T03:20:57.166 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[79819]: cluster 2024-03-07T03:20:57.066929+0000 mon.a (mon.0) 539 : cluster 0 osdmap e469: 8 total, 7 up, 8 in 2024-03-07T03:20:57.167 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.167 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:56 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-07T03:20:57.167 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:56 smithi148 systemd[1]: Stopped Ceph osd.0 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:20:57.167 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.167 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.167 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:20:57] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2024-03-07T03:20:57.168 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:20:57.168 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[82222]: cluster 2024-03-07T03:20:56.136226+0000 mon.a (mon.0) 536 : cluster 0 osdmap e468: 8 total, 7 up, 8 in 2024-03-07T03:20:57.168 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[82222]: cluster 2024-03-07T03:20:56.280483+0000 mgr.y (mgr.34101) 177 : cluster 0 pgmap v297: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 7.0 KiB/s wr, 89 op/s; 47/309 objects degraded (15.210%) 2024-03-07T03:20:57.168 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[82222]: audit 2024-03-07T03:20:56.293840+0000 mon.c (mon.2) 13 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]: dispatch 2024-03-07T03:20:57.168 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[82222]: audit 2024-03-07T03:20:56.294518+0000 mon.a (mon.0) 537 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]: dispatch 2024-03-07T03:20:57.168 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[82222]: audit 2024-03-07T03:20:56.302575+0000 mon.a (mon.0) 538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]': finished 2024-03-07T03:20:57.168 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:57 smithi148 bash[82222]: cluster 2024-03-07T03:20:57.066929+0000 mon.a (mon.0) 539 : cluster 0 osdmap e469: 8 total, 7 up, 8 in 2024-03-07T03:20:57.539 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:57 smithi148 systemd[1]: Started Ceph osd.0 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:20:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:57 smithi195 bash[66542]: cluster 2024-03-07T03:20:56.136226+0000 mon.a (mon.0) 536 : cluster 0 osdmap e468: 8 total, 7 up, 8 in 2024-03-07T03:20:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:57 smithi195 bash[66542]: cluster 2024-03-07T03:20:56.280483+0000 mgr.y (mgr.34101) 177 : cluster 0 pgmap v297: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 7.0 KiB/s wr, 89 op/s; 47/309 objects degraded (15.210%) 2024-03-07T03:20:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:57 smithi195 bash[66542]: audit 2024-03-07T03:20:56.293840+0000 mon.c (mon.2) 13 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]: dispatch 2024-03-07T03:20:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:57 smithi195 bash[66542]: audit 2024-03-07T03:20:56.294518+0000 mon.a (mon.0) 537 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]: dispatch 2024-03-07T03:20:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:57 smithi195 bash[66542]: audit 2024-03-07T03:20:56.302575+0000 mon.a (mon.0) 538 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/886ec8ba-77de-44ae-9899-763ffb326c64"}]': finished 2024-03-07T03:20:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:57 smithi195 bash[66542]: cluster 2024-03-07T03:20:57.066929+0000 mon.a (mon.0) 539 : cluster 0 osdmap e469: 8 total, 7 up, 8 in 2024-03-07T03:20:57.669 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 3 2024-03-07T03:20:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:58 smithi195 bash[66542]: audit 2024-03-07T03:20:57.259782+0000 mon.a (mon.0) 540 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:58 smithi195 bash[66542]: audit 2024-03-07T03:20:57.267318+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:58 smithi195 bash[66542]: audit 2024-03-07T03:20:57.279545+0000 mon.a (mon.0) 542 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:58 smithi195 bash[66542]: audit 2024-03-07T03:20:57.310016+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:58 smithi195 bash[66542]: audit 2024-03-07T03:20:57.311027+0000 mon.a (mon.0) 544 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:58.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:58 smithi195 bash[66542]: audit 2024-03-07T03:20:57.667286+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.148:0/1052953756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:20:58.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:58 smithi195 bash[66542]: cluster 2024-03-07T03:20:58.071624+0000 mon.a (mon.0) 545 : cluster 0 osdmap e470: 8 total, 7 up, 8 in 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[79819]: audit 2024-03-07T03:20:57.259782+0000 mon.a (mon.0) 540 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[79819]: audit 2024-03-07T03:20:57.267318+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[79819]: audit 2024-03-07T03:20:57.279545+0000 mon.a (mon.0) 542 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[79819]: audit 2024-03-07T03:20:57.310016+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[79819]: audit 2024-03-07T03:20:57.311027+0000 mon.a (mon.0) 544 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[79819]: audit 2024-03-07T03:20:57.667286+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.148:0/1052953756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[79819]: cluster 2024-03-07T03:20:58.071624+0000 mon.a (mon.0) 545 : cluster 0 osdmap e470: 8 total, 7 up, 8 in 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[82222]: audit 2024-03-07T03:20:57.259782+0000 mon.a (mon.0) 540 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[82222]: audit 2024-03-07T03:20:57.267318+0000 mon.a (mon.0) 541 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.608 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[82222]: audit 2024-03-07T03:20:57.279545+0000 mon.a (mon.0) 542 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:20:58.609 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[82222]: audit 2024-03-07T03:20:57.310016+0000 mon.a (mon.0) 543 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:20:58.609 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[82222]: audit 2024-03-07T03:20:57.311027+0000 mon.a (mon.0) 544 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:20:58.609 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[82222]: audit 2024-03-07T03:20:57.667286+0000 mon.c (mon.2) 14 : audit 0 from='client.? 172.21.15.148:0/1052953756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-07T03:20:58.609 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:58 smithi148 bash[82222]: cluster 2024-03-07T03:20:58.071624+0000 mon.a (mon.0) 545 : cluster 0 osdmap e470: 8 total, 7 up, 8 in 2024-03-07T03:20:59.129 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:20:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[79819]: cluster 2024-03-07T03:20:58.281559+0000 mgr.y (mgr.34101) 178 : cluster 0 pgmap v300: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 116 op/s; 50/315 objects degraded (15.873%) 2024-03-07T03:20:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[79819]: cluster 2024-03-07T03:20:59.075210+0000 mon.a (mon.0) 546 : cluster 0 osdmap e471: 8 total, 7 up, 8 in 2024-03-07T03:20:59.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[82222]: cluster 2024-03-07T03:20:58.281559+0000 mgr.y (mgr.34101) 178 : cluster 0 pgmap v300: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 116 op/s; 50/315 objects degraded (15.873%) 2024-03-07T03:20:59.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[82222]: cluster 2024-03-07T03:20:59.075210+0000 mon.a (mon.0) 546 : cluster 0 osdmap e471: 8 total, 7 up, 8 in 2024-03-07T03:20:59.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:59 smithi195 bash[66542]: cluster 2024-03-07T03:20:58.281559+0000 mgr.y (mgr.34101) 178 : cluster 0 pgmap v300: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 116 op/s; 50/315 objects degraded (15.873%) 2024-03-07T03:20:59.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:20:59 smithi195 bash[66542]: cluster 2024-03-07T03:20:59.075210+0000 mon.a (mon.0) 546 : cluster 0 osdmap e471: 8 total, 7 up, 8 in 2024-03-07T03:21:00.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[88638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-07T03:21:00.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[88638]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-07T03:21:00.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[88638]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-07T03:21:00.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[88638]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-07T03:21:00.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[88638]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-07T03:21:00.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[88638]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-07T03:21:00.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:20:59 smithi148 bash[88638]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-07T03:21:01.539 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:21:01 smithi148 bash[88937]: debug 2024-03-07T03:21:01.306+0000 7fb50c249700 1 -- 172.21.15.148:0/274559955 <== mon.2 v2:172.21.15.148:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fb5eb5d680 con 0x55fb5eb99000 2024-03-07T03:21:01.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:01 smithi148 bash[79819]: cluster 2024-03-07T03:21:00.084412+0000 mon.a (mon.0) 547 : cluster 0 osdmap e472: 8 total, 7 up, 8 in 2024-03-07T03:21:01.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:01 smithi148 bash[79819]: cluster 2024-03-07T03:21:00.282514+0000 mgr.y (mgr.34101) 179 : cluster 0 pgmap v303: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 116 op/s; 50/315 objects degraded (15.873%) 2024-03-07T03:21:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:01 smithi148 bash[82222]: cluster 2024-03-07T03:21:00.084412+0000 mon.a (mon.0) 547 : cluster 0 osdmap e472: 8 total, 7 up, 8 in 2024-03-07T03:21:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:01 smithi148 bash[82222]: cluster 2024-03-07T03:21:00.282514+0000 mgr.y (mgr.34101) 179 : cluster 0 pgmap v303: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 116 op/s; 50/315 objects degraded (15.873%) 2024-03-07T03:21:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:01 smithi195 bash[66542]: cluster 2024-03-07T03:21:00.084412+0000 mon.a (mon.0) 547 : cluster 0 osdmap e472: 8 total, 7 up, 8 in 2024-03-07T03:21:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:01 smithi195 bash[66542]: cluster 2024-03-07T03:21:00.282514+0000 mgr.y (mgr.34101) 179 : cluster 0 pgmap v303: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s rd, 9.7 KiB/s wr, 116 op/s; 50/315 objects degraded (15.873%) 2024-03-07T03:21:02.289 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:21:01 smithi148 bash[88937]: debug 2024-03-07T03:21:01.854+0000 7fb5153e17c0 -1 Falling back to public interface 2024-03-07T03:21:02.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[82222]: cluster 2024-03-07T03:21:01.087593+0000 mon.a (mon.0) 548 : cluster 0 osdmap e473: 8 total, 7 up, 8 in 2024-03-07T03:21:02.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[82222]: audit 2024-03-07T03:21:01.124638+0000 mon.c (mon.2) 15 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]: dispatch 2024-03-07T03:21:02.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[82222]: audit 2024-03-07T03:21:01.125037+0000 mon.a (mon.0) 549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]: dispatch 2024-03-07T03:21:02.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[82222]: audit 2024-03-07T03:21:01.128771+0000 mon.a (mon.0) 550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]': finished 2024-03-07T03:21:02.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[82222]: cluster 2024-03-07T03:21:02.062785+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 50/315 objects degraded (15.873%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[79819]: cluster 2024-03-07T03:21:01.087593+0000 mon.a (mon.0) 548 : cluster 0 osdmap e473: 8 total, 7 up, 8 in 2024-03-07T03:21:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[79819]: audit 2024-03-07T03:21:01.124638+0000 mon.c (mon.2) 15 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]: dispatch 2024-03-07T03:21:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[79819]: audit 2024-03-07T03:21:01.125037+0000 mon.a (mon.0) 549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]: dispatch 2024-03-07T03:21:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[79819]: audit 2024-03-07T03:21:01.128771+0000 mon.a (mon.0) 550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]': finished 2024-03-07T03:21:02.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:02 smithi148 bash[79819]: cluster 2024-03-07T03:21:02.062785+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 50/315 objects degraded (15.873%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:02.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:21:02 smithi195 bash[59134]: ts=2024-03-07T03:21:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:02 smithi195 bash[66542]: cluster 2024-03-07T03:21:01.087593+0000 mon.a (mon.0) 548 : cluster 0 osdmap e473: 8 total, 7 up, 8 in 2024-03-07T03:21:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:02 smithi195 bash[66542]: audit 2024-03-07T03:21:01.124638+0000 mon.c (mon.2) 15 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]: dispatch 2024-03-07T03:21:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:02 smithi195 bash[66542]: audit 2024-03-07T03:21:01.125037+0000 mon.a (mon.0) 549 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]: dispatch 2024-03-07T03:21:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:02 smithi195 bash[66542]: audit 2024-03-07T03:21:01.128771+0000 mon.a (mon.0) 550 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/d282a6a6-39ae-4e69-95a5-9370126216ad"}]': finished 2024-03-07T03:21:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:02 smithi195 bash[66542]: cluster 2024-03-07T03:21:02.062785+0000 mon.a (mon.0) 551 : cluster 3 Health check update: Degraded data redundancy: 50/315 objects degraded (15.873%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:03.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:03 smithi148 bash[79819]: cluster 2024-03-07T03:21:02.137212+0000 mon.a (mon.0) 552 : cluster 0 osdmap e474: 8 total, 7 up, 8 in 2024-03-07T03:21:03.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:03 smithi148 bash[79819]: cluster 2024-03-07T03:21:02.283494+0000 mgr.y (mgr.34101) 180 : cluster 0 pgmap v306: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 50/315 objects degraded (15.873%) 2024-03-07T03:21:03.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:03 smithi148 bash[82222]: cluster 2024-03-07T03:21:02.137212+0000 mon.a (mon.0) 552 : cluster 0 osdmap e474: 8 total, 7 up, 8 in 2024-03-07T03:21:03.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:03 smithi148 bash[82222]: cluster 2024-03-07T03:21:02.283494+0000 mgr.y (mgr.34101) 180 : cluster 0 pgmap v306: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 50/315 objects degraded (15.873%) 2024-03-07T03:21:03.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:03 smithi195 bash[66542]: cluster 2024-03-07T03:21:02.137212+0000 mon.a (mon.0) 552 : cluster 0 osdmap e474: 8 total, 7 up, 8 in 2024-03-07T03:21:03.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:03 smithi195 bash[66542]: cluster 2024-03-07T03:21:02.283494+0000 mgr.y (mgr.34101) 180 : cluster 0 pgmap v306: 105 pgs: 20 active+undersized+degraded, 22 active+undersized, 63 active+clean; 583 KiB data, 423 MiB used, 715 GiB / 715 GiB avail; 50/315 objects degraded (15.873%) 2024-03-07T03:21:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:04 smithi148 bash[79819]: cluster 2024-03-07T03:21:03.142610+0000 mon.a (mon.0) 553 : cluster 0 osdmap e475: 8 total, 7 up, 8 in 2024-03-07T03:21:04.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:04 smithi148 bash[79819]: audit 2024-03-07T03:21:03.819243+0000 mon.a (mon.0) 554 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:04.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:04 smithi148 bash[79819]: audit 2024-03-07T03:21:03.825240+0000 mon.a (mon.0) 555 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:04.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:04 smithi148 bash[82222]: cluster 2024-03-07T03:21:03.142610+0000 mon.a (mon.0) 553 : cluster 0 osdmap e475: 8 total, 7 up, 8 in 2024-03-07T03:21:04.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:04 smithi148 bash[82222]: audit 2024-03-07T03:21:03.819243+0000 mon.a (mon.0) 554 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:04.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:04 smithi148 bash[82222]: audit 2024-03-07T03:21:03.825240+0000 mon.a (mon.0) 555 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:04.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:04 smithi195 bash[66542]: cluster 2024-03-07T03:21:03.142610+0000 mon.a (mon.0) 553 : cluster 0 osdmap e475: 8 total, 7 up, 8 in 2024-03-07T03:21:04.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:04 smithi195 bash[66542]: audit 2024-03-07T03:21:03.819243+0000 mon.a (mon.0) 554 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:04.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:04 smithi195 bash[66542]: audit 2024-03-07T03:21:03.825240+0000 mon.a (mon.0) 555 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:05.092 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:21:05.289 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[88937]: debug 2024-03-07T03:21:05.025+0000 7fb5153e17c0 -1 osd.0 463 log_to_monitors true 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[79819]: cluster 2024-03-07T03:21:04.152782+0000 mon.a (mon.0) 556 : cluster 0 osdmap e476: 8 total, 7 up, 8 in 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[79819]: cluster 2024-03-07T03:21:04.284031+0000 mgr.y (mgr.34101) 181 : cluster 0 pgmap v309: 105 pgs: 21 active+undersized+degraded, 21 active+undersized, 63 active+clean; 583 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 11 KiB/s wr, 232 op/s; 53/333 objects degraded (15.916%) 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[79819]: audit 2024-03-07T03:21:04.764944+0000 mon.a (mon.0) 557 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[79819]: audit 2024-03-07T03:21:04.774164+0000 mon.a (mon.0) 558 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[79819]: audit 2024-03-07T03:21:05.033200+0000 mon.a (mon.0) 559 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[82222]: cluster 2024-03-07T03:21:04.152782+0000 mon.a (mon.0) 556 : cluster 0 osdmap e476: 8 total, 7 up, 8 in 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[82222]: cluster 2024-03-07T03:21:04.284031+0000 mgr.y (mgr.34101) 181 : cluster 0 pgmap v309: 105 pgs: 21 active+undersized+degraded, 21 active+undersized, 63 active+clean; 583 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 11 KiB/s wr, 232 op/s; 53/333 objects degraded (15.916%) 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[82222]: audit 2024-03-07T03:21:04.764944+0000 mon.a (mon.0) 557 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[82222]: audit 2024-03-07T03:21:04.774164+0000 mon.a (mon.0) 558 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:05.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[82222]: audit 2024-03-07T03:21:05.033200+0000 mon.a (mon.0) 559 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:21:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:05 smithi195 bash[66542]: cluster 2024-03-07T03:21:04.152782+0000 mon.a (mon.0) 556 : cluster 0 osdmap e476: 8 total, 7 up, 8 in 2024-03-07T03:21:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:05 smithi195 bash[66542]: cluster 2024-03-07T03:21:04.284031+0000 mgr.y (mgr.34101) 181 : cluster 0 pgmap v309: 105 pgs: 21 active+undersized+degraded, 21 active+undersized, 63 active+clean; 583 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 11 KiB/s wr, 232 op/s; 53/333 objects degraded (15.916%) 2024-03-07T03:21:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:05 smithi195 bash[66542]: audit 2024-03-07T03:21:04.764944+0000 mon.a (mon.0) 557 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:05 smithi195 bash[66542]: audit 2024-03-07T03:21:04.774164+0000 mon.a (mon.0) 558 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:05 smithi195 bash[66542]: audit 2024-03-07T03:21:05.033200+0000 mon.a (mon.0) 559 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-07T03:21:06.039 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:21:05 smithi148 bash[88937]: debug 2024-03-07T03:21:05.693+0000 7fb502831700 -1 osd.0 463 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:21:06.524 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[82222]: audit 2024-03-07T03:21:05.153291+0000 mon.a (mon.0) 560 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:21:06.524 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[82222]: cluster 2024-03-07T03:21:05.163986+0000 mon.a (mon.0) 561 : cluster 0 osdmap e477: 8 total, 7 up, 8 in 2024-03-07T03:21:06.524 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[82222]: audit 2024-03-07T03:21:05.164469+0000 mon.a (mon.0) 562 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:21:06.524 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[82222]: audit 2024-03-07T03:21:05.678506+0000 mon.a (mon.0) 563 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' 2024-03-07T03:21:06.525 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[79819]: debug 2024-03-07T03:21:06.149+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:06.525 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[79819]: audit 2024-03-07T03:21:05.153291+0000 mon.a (mon.0) 560 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:21:06.525 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[79819]: cluster 2024-03-07T03:21:05.163986+0000 mon.a (mon.0) 561 : cluster 0 osdmap e477: 8 total, 7 up, 8 in 2024-03-07T03:21:06.525 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[79819]: audit 2024-03-07T03:21:05.164469+0000 mon.a (mon.0) 562 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:21:06.525 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:06 smithi148 bash[79819]: audit 2024-03-07T03:21:05.678506+0000 mon.a (mon.0) 563 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' 2024-03-07T03:21:06.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:06 smithi195 bash[66542]: audit 2024-03-07T03:21:05.153291+0000 mon.a (mon.0) 560 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-07T03:21:06.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:06 smithi195 bash[66542]: cluster 2024-03-07T03:21:05.163986+0000 mon.a (mon.0) 561 : cluster 0 osdmap e477: 8 total, 7 up, 8 in 2024-03-07T03:21:06.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:06 smithi195 bash[66542]: audit 2024-03-07T03:21:05.164469+0000 mon.a (mon.0) 562 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:21:06.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:06 smithi195 bash[66542]: audit 2024-03-07T03:21:05.678506+0000 mon.a (mon.0) 563 : audit 1 from='osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114]' entity='osd.0' 2024-03-07T03:21:07.321 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (11017 ms) 2024-03-07T03:21:07.321 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-03-07T03:21:07.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:21:07] "GET /metrics HTTP/1.1" 200 36521 "" "Prometheus/2.43.0" 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[82222]: cluster 2024-03-07T03:21:06.155067+0000 mon.a (mon.0) 564 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[82222]: cluster 2024-03-07T03:21:06.155150+0000 mon.a (mon.0) 565 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[82222]: cluster 2024-03-07T03:21:06.169840+0000 mon.a (mon.0) 566 : cluster 1 osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114] boot 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[82222]: cluster 2024-03-07T03:21:06.169967+0000 mon.a (mon.0) 567 : cluster 0 osdmap e478: 8 total, 8 up, 8 in 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[82222]: audit 2024-03-07T03:21:06.170554+0000 mon.a (mon.0) 568 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[82222]: cluster 2024-03-07T03:21:06.284560+0000 mgr.y (mgr.34101) 182 : cluster 0 pgmap v312: 105 pgs: 21 active+undersized+degraded, 21 active+undersized, 63 active+clean; 583 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 11 KiB/s wr, 232 op/s; 53/333 objects degraded (15.916%) 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[82222]: cluster 2024-03-07T03:21:07.064121+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 53/333 objects degraded (15.916%), 21 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[79819]: cluster 2024-03-07T03:21:06.155067+0000 mon.a (mon.0) 564 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[79819]: cluster 2024-03-07T03:21:06.155150+0000 mon.a (mon.0) 565 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[79819]: cluster 2024-03-07T03:21:06.169840+0000 mon.a (mon.0) 566 : cluster 1 osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114] boot 2024-03-07T03:21:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[79819]: cluster 2024-03-07T03:21:06.169967+0000 mon.a (mon.0) 567 : cluster 0 osdmap e478: 8 total, 8 up, 8 in 2024-03-07T03:21:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[79819]: audit 2024-03-07T03:21:06.170554+0000 mon.a (mon.0) 568 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:21:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[79819]: cluster 2024-03-07T03:21:06.284560+0000 mgr.y (mgr.34101) 182 : cluster 0 pgmap v312: 105 pgs: 21 active+undersized+degraded, 21 active+undersized, 63 active+clean; 583 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 11 KiB/s wr, 232 op/s; 53/333 objects degraded (15.916%) 2024-03-07T03:21:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:07 smithi148 bash[79819]: cluster 2024-03-07T03:21:07.064121+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 53/333 objects degraded (15.916%), 21 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:07 smithi195 bash[66542]: cluster 2024-03-07T03:21:06.155067+0000 mon.a (mon.0) 564 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:07 smithi195 bash[66542]: cluster 2024-03-07T03:21:06.155150+0000 mon.a (mon.0) 565 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:07 smithi195 bash[66542]: cluster 2024-03-07T03:21:06.169840+0000 mon.a (mon.0) 566 : cluster 1 osd.0 [v2:172.21.15.148:6802/3706550114,v1:172.21.15.148:6803/3706550114] boot 2024-03-07T03:21:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:07 smithi195 bash[66542]: cluster 2024-03-07T03:21:06.169967+0000 mon.a (mon.0) 567 : cluster 0 osdmap e478: 8 total, 8 up, 8 in 2024-03-07T03:21:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:07 smithi195 bash[66542]: audit 2024-03-07T03:21:06.170554+0000 mon.a (mon.0) 568 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-07T03:21:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:07 smithi195 bash[66542]: cluster 2024-03-07T03:21:06.284560+0000 mgr.y (mgr.34101) 182 : cluster 0 pgmap v312: 105 pgs: 21 active+undersized+degraded, 21 active+undersized, 63 active+clean; 583 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 11 KiB/s wr, 232 op/s; 53/333 objects degraded (15.916%) 2024-03-07T03:21:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:07 smithi195 bash[66542]: cluster 2024-03-07T03:21:07.064121+0000 mon.a (mon.0) 569 : cluster 3 Health check update: Degraded data redundancy: 53/333 objects degraded (15.916%), 21 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[79819]: cluster 2024-03-07T03:21:07.178122+0000 mon.a (mon.0) 570 : cluster 0 osdmap e479: 8 total, 8 up, 8 in 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[79819]: audit 2024-03-07T03:21:07.311214+0000 mon.c (mon.2) 16 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]: dispatch 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[79819]: audit 2024-03-07T03:21:07.311600+0000 mon.a (mon.0) 571 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]: dispatch 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[79819]: audit 2024-03-07T03:21:07.315121+0000 mon.a (mon.0) 572 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]': finished 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[79819]: audit 2024-03-07T03:21:07.320344+0000 mon.c (mon.2) 17 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]: dispatch 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[79819]: audit 2024-03-07T03:21:07.320668+0000 mon.a (mon.0) 573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]: dispatch 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[79819]: audit 2024-03-07T03:21:07.324075+0000 mon.a (mon.0) 574 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]': finished 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[82222]: cluster 2024-03-07T03:21:07.178122+0000 mon.a (mon.0) 570 : cluster 0 osdmap e479: 8 total, 8 up, 8 in 2024-03-07T03:21:08.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[82222]: audit 2024-03-07T03:21:07.311214+0000 mon.c (mon.2) 16 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]: dispatch 2024-03-07T03:21:08.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[82222]: audit 2024-03-07T03:21:07.311600+0000 mon.a (mon.0) 571 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]: dispatch 2024-03-07T03:21:08.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[82222]: audit 2024-03-07T03:21:07.315121+0000 mon.a (mon.0) 572 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]': finished 2024-03-07T03:21:08.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[82222]: audit 2024-03-07T03:21:07.320344+0000 mon.c (mon.2) 17 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]: dispatch 2024-03-07T03:21:08.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[82222]: audit 2024-03-07T03:21:07.320668+0000 mon.a (mon.0) 573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]: dispatch 2024-03-07T03:21:08.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:08 smithi148 bash[82222]: audit 2024-03-07T03:21:07.324075+0000 mon.a (mon.0) 574 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]': finished 2024-03-07T03:21:08.484 INFO:teuthology.orchestra.run.smithi148.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:21:08.484 INFO:teuthology.orchestra.run.smithi148.stdout:alertmanager.a smithi148 *:9093,9094 running (11m) 4s ago 11m 14.3M - 0.25.0 c8568f914cd2 a6ee726aca40 2024-03-07T03:21:08.484 INFO:teuthology.orchestra.run.smithi148.stdout:grafana.a smithi195 *:3000 running (11m) 25s ago 11m 79.5M - 9.4.7 954c08fa6188 a09a8cb7fec4 2024-03-07T03:21:08.484 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi148.qizikc smithi148 running (9m) 4s ago 9m 15.3M - 18.2.1-718-gae75fc9d 0164bb8a7c67 d82cf82f7898 2024-03-07T03:21:08.484 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi195.gtlmjn smithi195 running (9m) 25s ago 9m 18.4M - 18.2.1-718-gae75fc9d 0164bb8a7c67 f9ba5c67a194 2024-03-07T03:21:08.484 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.x smithi195 *:8443,9283,8765 running (4m) 25s ago 19m 439M - 19.0.0-1905-gea9a5688 fdb83767c1ab 758d97044603 2024-03-07T03:21:08.484 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.y smithi148 *:8443,9283,8765 running (5m) 4s ago 22m 495M - 19.0.0-1905-gea9a5688 fdb83767c1ab 85cf9ffcb609 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:mon.a smithi148 running (4m) 4s ago 22m 53.9M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 39441e18edee 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:mon.b smithi195 running (111s) 25s ago 20m 34.5M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 13f5d79dedd1 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:mon.c smithi148 running (2m) 4s ago 20m 39.2M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab a8f98b5ace69 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.a smithi148 *:9100 running (12m) 4s ago 12m 8588k - 1.5.0 0da6a335fe13 d9b895f99d5c 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.b smithi195 *:9100 running (12m) 25s ago 12m 8712k - 1.5.0 0da6a335fe13 be49ab11d18e 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.0 smithi148 running (7s) 4s ago 19m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab d8d07b2483c5 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.1 smithi148 running (18m) 4s ago 18m 127M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 3125450ae62f 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.2 smithi148 running (17m) 4s ago 17m 141M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 b463a715bf36 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.3 smithi148 running (16m) 4s ago 16m 129M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 180f02eefffa 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.4 smithi195 running (15m) 25s ago 15m 127M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 5023ce93366a 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.5 smithi195 running (15m) 25s ago 15m 104M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c850d7f8ae40 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.6 smithi195 running (14m) 25s ago 14m 142M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 332f73f59fdc 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:osd.7 smithi195 running (13m) 25s ago 13m 110M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9a7e2329950b 2024-03-07T03:21:08.485 INFO:teuthology.orchestra.run.smithi148.stdout:prometheus.a smithi195 *:9095 running (4m) 25s ago 12m 49.9M - 2.43.0 a07b618ecd1d e5c6727b35a4 2024-03-07T03:21:08.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:08 smithi195 bash[66542]: cluster 2024-03-07T03:21:07.178122+0000 mon.a (mon.0) 570 : cluster 0 osdmap e479: 8 total, 8 up, 8 in 2024-03-07T03:21:08.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:08 smithi195 bash[66542]: audit 2024-03-07T03:21:07.311214+0000 mon.c (mon.2) 16 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]: dispatch 2024-03-07T03:21:08.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:08 smithi195 bash[66542]: audit 2024-03-07T03:21:07.311600+0000 mon.a (mon.0) 571 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]: dispatch 2024-03-07T03:21:08.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:08 smithi195 bash[66542]: audit 2024-03-07T03:21:07.315121+0000 mon.a (mon.0) 572 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/883618f0-a275-404f-9836-c0d6be7dd8ef"}]': finished 2024-03-07T03:21:08.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:08 smithi195 bash[66542]: audit 2024-03-07T03:21:07.320344+0000 mon.c (mon.2) 17 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]: dispatch 2024-03-07T03:21:08.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:08 smithi195 bash[66542]: audit 2024-03-07T03:21:07.320668+0000 mon.a (mon.0) 573 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]: dispatch 2024-03-07T03:21:08.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:08 smithi195 bash[66542]: audit 2024-03-07T03:21:07.324075+0000 mon.a (mon.0) 574 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/dc3efdea-3447-47ba-b355-0906a1279807"}]': finished 2024-03-07T03:21:09.277 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:21:09.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:09 smithi148 bash[79819]: cluster 2024-03-07T03:21:08.285412+0000 mgr.y (mgr.34101) 183 : cluster 0 pgmap v314: 105 pgs: 9 active+undersized+degraded, 9 active+undersized, 87 active+clean; 583 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 4.8 KiB/s wr, 83 op/s; 24/309 objects degraded (7.767%); 0 B/s, 2 keys/s, 1 objects/s recovering 2024-03-07T03:21:09.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:09 smithi148 bash[79819]: cluster 2024-03-07T03:21:08.330023+0000 mon.a (mon.0) 575 : cluster 0 osdmap e480: 8 total, 8 up, 8 in 2024-03-07T03:21:09.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:09 smithi148 bash[79819]: audit 2024-03-07T03:21:08.457987+0000 mgr.y (mgr.34101) 184 : audit 0 from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:21:09.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:09 smithi148 bash[82222]: cluster 2024-03-07T03:21:08.285412+0000 mgr.y (mgr.34101) 183 : cluster 0 pgmap v314: 105 pgs: 9 active+undersized+degraded, 9 active+undersized, 87 active+clean; 583 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 4.8 KiB/s wr, 83 op/s; 24/309 objects degraded (7.767%); 0 B/s, 2 keys/s, 1 objects/s recovering 2024-03-07T03:21:09.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:09 smithi148 bash[82222]: cluster 2024-03-07T03:21:08.330023+0000 mon.a (mon.0) 575 : cluster 0 osdmap e480: 8 total, 8 up, 8 in 2024-03-07T03:21:09.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:09 smithi148 bash[82222]: audit 2024-03-07T03:21:08.457987+0000 mgr.y (mgr.34101) 184 : audit 0 from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:21:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:09 smithi195 bash[66542]: cluster 2024-03-07T03:21:08.285412+0000 mgr.y (mgr.34101) 183 : cluster 0 pgmap v314: 105 pgs: 9 active+undersized+degraded, 9 active+undersized, 87 active+clean; 583 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 4.8 KiB/s wr, 83 op/s; 24/309 objects degraded (7.767%); 0 B/s, 2 keys/s, 1 objects/s recovering 2024-03-07T03:21:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:09 smithi195 bash[66542]: cluster 2024-03-07T03:21:08.330023+0000 mon.a (mon.0) 575 : cluster 0 osdmap e480: 8 total, 8 up, 8 in 2024-03-07T03:21:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:09 smithi195 bash[66542]: audit 2024-03-07T03:21:08.457987+0000 mgr.y (mgr.34101) 184 : audit 0 from='client.44162 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:21:10.691 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:10 smithi148 bash[79819]: cluster 2024-03-07T03:21:09.342113+0000 mon.a (mon.0) 576 : cluster 0 osdmap e481: 8 total, 8 up, 8 in 2024-03-07T03:21:10.691 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:10 smithi148 bash[82222]: cluster 2024-03-07T03:21:09.342113+0000 mon.a (mon.0) 576 : cluster 0 osdmap e481: 8 total, 8 up, 8 in 2024-03-07T03:21:10.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:10 smithi195 bash[66542]: cluster 2024-03-07T03:21:09.342113+0000 mon.a (mon.0) 576 : cluster 0 osdmap e481: 8 total, 8 up, 8 in 2024-03-07T03:21:11.457 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.SnapshotImageState (4136 ms) 2024-03-07T03:21:11.457 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-03-07T03:21:11.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:11 smithi148 bash[79819]: cluster 2024-03-07T03:21:10.286229+0000 mgr.y (mgr.34101) 185 : cluster 0 pgmap v317: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 8.0 KiB/s wr, 98 op/s; 3 B/s, 2 keys/s, 3 objects/s recovering 2024-03-07T03:21:11.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:11 smithi148 bash[79819]: cluster 2024-03-07T03:21:10.335897+0000 mon.a (mon.0) 577 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/309 objects degraded (7.767%), 9 pgs degraded) 2024-03-07T03:21:11.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:11 smithi148 bash[79819]: cluster 2024-03-07T03:21:10.355860+0000 mon.a (mon.0) 578 : cluster 0 osdmap e482: 8 total, 8 up, 8 in 2024-03-07T03:21:11.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:11 smithi148 bash[82222]: cluster 2024-03-07T03:21:10.286229+0000 mgr.y (mgr.34101) 185 : cluster 0 pgmap v317: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 8.0 KiB/s wr, 98 op/s; 3 B/s, 2 keys/s, 3 objects/s recovering 2024-03-07T03:21:11.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:11 smithi148 bash[82222]: cluster 2024-03-07T03:21:10.335897+0000 mon.a (mon.0) 577 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/309 objects degraded (7.767%), 9 pgs degraded) 2024-03-07T03:21:11.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:11 smithi148 bash[82222]: cluster 2024-03-07T03:21:10.355860+0000 mon.a (mon.0) 578 : cluster 0 osdmap e482: 8 total, 8 up, 8 in 2024-03-07T03:21:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:11 smithi195 bash[66542]: cluster 2024-03-07T03:21:10.286229+0000 mgr.y (mgr.34101) 185 : cluster 0 pgmap v317: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s rd, 8.0 KiB/s wr, 98 op/s; 3 B/s, 2 keys/s, 3 objects/s recovering 2024-03-07T03:21:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:11 smithi195 bash[66542]: cluster 2024-03-07T03:21:10.335897+0000 mon.a (mon.0) 577 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 24/309 objects degraded (7.767%), 9 pgs degraded) 2024-03-07T03:21:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:11 smithi195 bash[66542]: cluster 2024-03-07T03:21:10.355860+0000 mon.a (mon.0) 578 : cluster 0 osdmap e482: 8 total, 8 up, 8 in 2024-03-07T03:21:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:21:12 smithi195 bash[59134]: ts=2024-03-07T03:21:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:12 smithi195 bash[66542]: cluster 2024-03-07T03:21:11.365717+0000 mon.a (mon.0) 579 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2024-03-07T03:21:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:12 smithi195 bash[66542]: cluster 2024-03-07T03:21:12.071539+0000 mon.a (mon.0) 580 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2024-03-07T03:21:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:12 smithi195 bash[66542]: audit 2024-03-07T03:21:12.317240+0000 mon.a (mon.0) 581 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:12 smithi195 bash[66542]: audit 2024-03-07T03:21:12.325843+0000 mon.a (mon.0) 582 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[82222]: cluster 2024-03-07T03:21:11.365717+0000 mon.a (mon.0) 579 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2024-03-07T03:21:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[82222]: cluster 2024-03-07T03:21:12.071539+0000 mon.a (mon.0) 580 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2024-03-07T03:21:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[82222]: audit 2024-03-07T03:21:12.317240+0000 mon.a (mon.0) 581 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[82222]: audit 2024-03-07T03:21:12.325843+0000 mon.a (mon.0) 582 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[79819]: cluster 2024-03-07T03:21:11.365717+0000 mon.a (mon.0) 579 : cluster 0 osdmap e483: 8 total, 8 up, 8 in 2024-03-07T03:21:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[79819]: cluster 2024-03-07T03:21:12.071539+0000 mon.a (mon.0) 580 : cluster 0 osdmap e484: 8 total, 8 up, 8 in 2024-03-07T03:21:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[79819]: audit 2024-03-07T03:21:12.317240+0000 mon.a (mon.0) 581 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:12 smithi148 bash[79819]: audit 2024-03-07T03:21:12.325843+0000 mon.a (mon.0) 582 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:13.640 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:13 smithi148 bash[79819]: cluster 2024-03-07T03:21:12.286875+0000 mgr.y (mgr.34101) 186 : cluster 0 pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.3 KiB/s wr, 15 op/s; 4 B/s, 1 objects/s recovering 2024-03-07T03:21:13.640 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:13 smithi148 bash[79819]: cluster 2024-03-07T03:21:13.079790+0000 mon.a (mon.0) 583 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2024-03-07T03:21:13.640 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:13 smithi148 bash[82222]: cluster 2024-03-07T03:21:12.286875+0000 mgr.y (mgr.34101) 186 : cluster 0 pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.3 KiB/s wr, 15 op/s; 4 B/s, 1 objects/s recovering 2024-03-07T03:21:13.641 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:13 smithi148 bash[82222]: cluster 2024-03-07T03:21:13.079790+0000 mon.a (mon.0) 583 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2024-03-07T03:21:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:13 smithi195 bash[66542]: cluster 2024-03-07T03:21:12.286875+0000 mgr.y (mgr.34101) 186 : cluster 0 pgmap v321: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 6.6 KiB/s rd, 3.3 KiB/s wr, 15 op/s; 4 B/s, 1 objects/s recovering 2024-03-07T03:21:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:13 smithi195 bash[66542]: cluster 2024-03-07T03:21:13.079790+0000 mon.a (mon.0) 583 : cluster 0 osdmap e485: 8 total, 8 up, 8 in 2024-03-07T03:21:14.581 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:21:15.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: cluster 2024-03-07T03:21:14.085601+0000 mon.a (mon.0) 584 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2024-03-07T03:21:15.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.224233+0000 mon.a (mon.0) 585 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.237252+0000 mon.a (mon.0) 586 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.239820+0000 mon.a (mon.0) 587 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:15.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.241720+0000 mon.a (mon.0) 588 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:15.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.254496+0000 mon.a (mon.0) 589 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: cluster 2024-03-07T03:21:14.287461+0000 mgr.y (mgr.34101) 187 : cluster 0 pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 24 KiB/s wr, 244 op/s 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.340108+0000 mon.a (mon.0) 590 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.344837+0000 mon.a (mon.0) 591 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.348899+0000 mon.a (mon.0) 592 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.351768+0000 mon.a (mon.0) 593 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.355184+0000 mon.a (mon.0) 594 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: audit 2024-03-07T03:21:14.355592+0000 mgr.y (mgr.34101) 188 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[79819]: cephadm 2024-03-07T03:21:14.357800+0000 mgr.y (mgr.34101) 189 : cephadm 1 Upgrade: osd.1 is safe to restart 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: cluster 2024-03-07T03:21:14.085601+0000 mon.a (mon.0) 584 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.224233+0000 mon.a (mon.0) 585 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.237252+0000 mon.a (mon.0) 586 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.239820+0000 mon.a (mon.0) 587 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.241720+0000 mon.a (mon.0) 588 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.254496+0000 mon.a (mon.0) 589 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: cluster 2024-03-07T03:21:14.287461+0000 mgr.y (mgr.34101) 187 : cluster 0 pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 24 KiB/s wr, 244 op/s 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.340108+0000 mon.a (mon.0) 590 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.344837+0000 mon.a (mon.0) 591 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.348899+0000 mon.a (mon.0) 592 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.351768+0000 mon.a (mon.0) 593 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.355184+0000 mon.a (mon.0) 594 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: audit 2024-03-07T03:21:14.355592+0000 mgr.y (mgr.34101) 188 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:21:15.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:15 smithi148 bash[82222]: cephadm 2024-03-07T03:21:14.357800+0000 mgr.y (mgr.34101) 189 : cephadm 1 Upgrade: osd.1 is safe to restart 2024-03-07T03:21:15.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: cluster 2024-03-07T03:21:14.085601+0000 mon.a (mon.0) 584 : cluster 0 osdmap e486: 8 total, 8 up, 8 in 2024-03-07T03:21:15.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.224233+0000 mon.a (mon.0) 585 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.237252+0000 mon.a (mon.0) 586 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.239820+0000 mon.a (mon.0) 587 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:15.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.241720+0000 mon.a (mon.0) 588 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:15.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.254496+0000 mon.a (mon.0) 589 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: cluster 2024-03-07T03:21:14.287461+0000 mgr.y (mgr.34101) 187 : cluster 0 pgmap v324: 105 pgs: 105 active+clean; 583 KiB data, 416 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 24 KiB/s wr, 244 op/s 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.340108+0000 mon.a (mon.0) 590 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.344837+0000 mon.a (mon.0) 591 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.348899+0000 mon.a (mon.0) 592 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.351768+0000 mon.a (mon.0) 593 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.355184+0000 mon.a (mon.0) 594 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: audit 2024-03-07T03:21:14.355592+0000 mgr.y (mgr.34101) 188 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-07T03:21:15.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:15 smithi195 bash[66542]: cephadm 2024-03-07T03:21:14.357800+0000 mgr.y (mgr.34101) 189 : cephadm 1 Upgrade: osd.1 is safe to restart 2024-03-07T03:21:16.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:16 smithi148 bash[79819]: cluster 2024-03-07T03:21:15.091843+0000 mon.a (mon.0) 595 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2024-03-07T03:21:16.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:16 smithi148 bash[82222]: cluster 2024-03-07T03:21:15.091843+0000 mon.a (mon.0) 595 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2024-03-07T03:21:16.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:16 smithi195 bash[66542]: cluster 2024-03-07T03:21:15.091843+0000 mon.a (mon.0) 595 : cluster 0 osdmap e487: 8 total, 8 up, 8 in 2024-03-07T03:21:17.486 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:17 smithi148 bash[79819]: cluster 2024-03-07T03:21:16.100331+0000 mon.a (mon.0) 596 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2024-03-07T03:21:17.486 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:17 smithi148 bash[79819]: cluster 2024-03-07T03:21:16.287936+0000 mgr.y (mgr.34101) 190 : cluster 0 pgmap v327: 105 pgs: 105 active+clean; 583 KiB data, 417 MiB used, 715 GiB / 715 GiB avail; 184 KiB/s rd, 25 KiB/s wr, 256 op/s 2024-03-07T03:21:17.486 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:21:17] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2024-03-07T03:21:17.487 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:17 smithi148 bash[82222]: cluster 2024-03-07T03:21:16.100331+0000 mon.a (mon.0) 596 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2024-03-07T03:21:17.487 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:17 smithi148 bash[82222]: cluster 2024-03-07T03:21:16.287936+0000 mgr.y (mgr.34101) 190 : cluster 0 pgmap v327: 105 pgs: 105 active+clean; 583 KiB data, 417 MiB used, 715 GiB / 715 GiB avail; 184 KiB/s rd, 25 KiB/s wr, 256 op/s 2024-03-07T03:21:17.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:17 smithi195 bash[66542]: cluster 2024-03-07T03:21:16.100331+0000 mon.a (mon.0) 596 : cluster 0 osdmap e488: 8 total, 8 up, 8 in 2024-03-07T03:21:17.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:17 smithi195 bash[66542]: cluster 2024-03-07T03:21:16.287936+0000 mgr.y (mgr.34101) 190 : cluster 0 pgmap v327: 105 pgs: 105 active+clean; 583 KiB data, 417 MiB used, 715 GiB / 715 GiB avail; 184 KiB/s rd, 25 KiB/s wr, 256 op/s 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[79819]: cluster 2024-03-07T03:21:17.115010+0000 mon.a (mon.0) 597 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[79819]: audit 2024-03-07T03:21:17.410188+0000 mon.a (mon.0) 598 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[79819]: cephadm 2024-03-07T03:21:17.531349+0000 mgr.y (mgr.34101) 191 : cephadm 1 Upgrade: Updating osd.1 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[79819]: audit 2024-03-07T03:21:17.538172+0000 mon.a (mon.0) 599 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[79819]: audit 2024-03-07T03:21:17.540588+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[79819]: audit 2024-03-07T03:21:17.541296+0000 mon.a (mon.0) 601 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[79819]: cephadm 2024-03-07T03:21:17.545863+0000 mgr.y (mgr.34101) 192 : cephadm 1 Deploying daemon osd.1 on smithi148 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[82222]: cluster 2024-03-07T03:21:17.115010+0000 mon.a (mon.0) 597 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2024-03-07T03:21:18.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[82222]: audit 2024-03-07T03:21:17.410188+0000 mon.a (mon.0) 598 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:18.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[82222]: cephadm 2024-03-07T03:21:17.531349+0000 mgr.y (mgr.34101) 191 : cephadm 1 Upgrade: Updating osd.1 2024-03-07T03:21:18.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[82222]: audit 2024-03-07T03:21:17.538172+0000 mon.a (mon.0) 599 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:18.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[82222]: audit 2024-03-07T03:21:17.540588+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:21:18.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[82222]: audit 2024-03-07T03:21:17.541296+0000 mon.a (mon.0) 601 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:18.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:18 smithi148 bash[82222]: cephadm 2024-03-07T03:21:17.545863+0000 mgr.y (mgr.34101) 192 : cephadm 1 Deploying daemon osd.1 on smithi148 2024-03-07T03:21:18.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:18 smithi195 bash[66542]: cluster 2024-03-07T03:21:17.115010+0000 mon.a (mon.0) 597 : cluster 0 osdmap e489: 8 total, 8 up, 8 in 2024-03-07T03:21:18.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:18 smithi195 bash[66542]: audit 2024-03-07T03:21:17.410188+0000 mon.a (mon.0) 598 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:18.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:18 smithi195 bash[66542]: cephadm 2024-03-07T03:21:17.531349+0000 mgr.y (mgr.34101) 191 : cephadm 1 Upgrade: Updating osd.1 2024-03-07T03:21:18.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:18 smithi195 bash[66542]: audit 2024-03-07T03:21:17.538172+0000 mon.a (mon.0) 599 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:18.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:18 smithi195 bash[66542]: audit 2024-03-07T03:21:17.540588+0000 mon.a (mon.0) 600 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-07T03:21:18.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:18 smithi195 bash[66542]: audit 2024-03-07T03:21:17.541296+0000 mon.a (mon.0) 601 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:18.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:18 smithi195 bash[66542]: cephadm 2024-03-07T03:21:17.545863+0000 mgr.y (mgr.34101) 192 : cephadm 1 Deploying daemon osd.1 on smithi148 2024-03-07T03:21:19.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:19 smithi148 bash[79819]: cluster 2024-03-07T03:21:18.118706+0000 mon.a (mon.0) 602 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2024-03-07T03:21:19.256 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:19 smithi148 bash[79819]: cluster 2024-03-07T03:21:18.289132+0000 mgr.y (mgr.34101) 193 : cluster 0 pgmap v330: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s rd, 8.2 KiB/s wr, 130 op/s 2024-03-07T03:21:19.256 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:19 smithi148 bash[82222]: cluster 2024-03-07T03:21:18.118706+0000 mon.a (mon.0) 602 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2024-03-07T03:21:19.256 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:19 smithi148 bash[82222]: cluster 2024-03-07T03:21:18.289132+0000 mgr.y (mgr.34101) 193 : cluster 0 pgmap v330: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s rd, 8.2 KiB/s wr, 130 op/s 2024-03-07T03:21:19.429 INFO:teuthology.orchestra.run.smithi148.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:21:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:19 smithi195 bash[66542]: cluster 2024-03-07T03:21:18.118706+0000 mon.a (mon.0) 602 : cluster 0 osdmap e490: 8 total, 8 up, 8 in 2024-03-07T03:21:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:19 smithi195 bash[66542]: cluster 2024-03-07T03:21:18.289132+0000 mgr.y (mgr.34101) 193 : cluster 0 pgmap v330: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 95 KiB/s rd, 8.2 KiB/s wr, 130 op/s 2024-03-07T03:21:20.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[79819]: cluster 2024-03-07T03:21:19.129058+0000 mon.a (mon.0) 603 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2024-03-07T03:21:20.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[79819]: audit 2024-03-07T03:21:19.434989+0000 mgr.y (mgr.34101) 194 : audit 0 from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:21:20.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[79819]: cephadm 2024-03-07T03:21:19.435552+0000 mgr.y (mgr.34101) 195 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:21:20.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[79819]: audit 2024-03-07T03:21:19.442323+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:20.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[82222]: cluster 2024-03-07T03:21:19.129058+0000 mon.a (mon.0) 603 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2024-03-07T03:21:20.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[82222]: audit 2024-03-07T03:21:19.434989+0000 mgr.y (mgr.34101) 194 : audit 0 from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:21:20.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[82222]: cephadm 2024-03-07T03:21:19.435552+0000 mgr.y (mgr.34101) 195 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:21:20.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:20 smithi148 bash[82222]: audit 2024-03-07T03:21:19.442323+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:20.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:20 smithi195 bash[66542]: cluster 2024-03-07T03:21:19.129058+0000 mon.a (mon.0) 603 : cluster 0 osdmap e491: 8 total, 8 up, 8 in 2024-03-07T03:21:20.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:20 smithi195 bash[66542]: audit 2024-03-07T03:21:19.434989+0000 mgr.y (mgr.34101) 194 : audit 0 from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:21:20.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:20 smithi195 bash[66542]: cephadm 2024-03-07T03:21:19.435552+0000 mgr.y (mgr.34101) 195 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:21:20.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:20 smithi195 bash[66542]: audit 2024-03-07T03:21:19.442323+0000 mon.a (mon.0) 604 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:21.059 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'sleep 60' 2024-03-07T03:21:21.228 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (9771 ms) 2024-03-07T03:21:21.228 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-03-07T03:21:21.391 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:21 smithi148 bash[79819]: cluster 2024-03-07T03:21:20.138647+0000 mon.a (mon.0) 605 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2024-03-07T03:21:21.391 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:21 smithi148 bash[79819]: cluster 2024-03-07T03:21:20.289937+0000 mgr.y (mgr.34101) 196 : cluster 0 pgmap v333: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 7.0 KiB/s wr, 124 op/s 2024-03-07T03:21:21.392 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:21 smithi148 bash[82222]: cluster 2024-03-07T03:21:20.138647+0000 mon.a (mon.0) 605 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2024-03-07T03:21:21.392 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:21 smithi148 bash[82222]: cluster 2024-03-07T03:21:20.289937+0000 mgr.y (mgr.34101) 196 : cluster 0 pgmap v333: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 7.0 KiB/s wr, 124 op/s 2024-03-07T03:21:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:21 smithi195 bash[66542]: cluster 2024-03-07T03:21:20.138647+0000 mon.a (mon.0) 605 : cluster 0 osdmap e492: 8 total, 8 up, 8 in 2024-03-07T03:21:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:21 smithi195 bash[66542]: cluster 2024-03-07T03:21:20.289937+0000 mgr.y (mgr.34101) 196 : cluster 0 pgmap v333: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 7.0 KiB/s wr, 124 op/s 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[79819]: cluster 2024-03-07T03:21:21.147523+0000 mon.a (mon.0) 606 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[79819]: audit 2024-03-07T03:21:21.227302+0000 mon.c (mon.2) 18 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]: dispatch 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[79819]: audit 2024-03-07T03:21:21.227903+0000 mon.a (mon.0) 607 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]: dispatch 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[79819]: audit 2024-03-07T03:21:21.231441+0000 mon.a (mon.0) 608 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]': finished 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[79819]: cluster 2024-03-07T03:21:22.078659+0000 mon.a (mon.0) 609 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[82222]: cluster 2024-03-07T03:21:21.147523+0000 mon.a (mon.0) 606 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[82222]: audit 2024-03-07T03:21:21.227302+0000 mon.c (mon.2) 18 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]: dispatch 2024-03-07T03:21:22.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[82222]: audit 2024-03-07T03:21:21.227903+0000 mon.a (mon.0) 607 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]: dispatch 2024-03-07T03:21:22.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[82222]: audit 2024-03-07T03:21:21.231441+0000 mon.a (mon.0) 608 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]': finished 2024-03-07T03:21:22.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:22 smithi148 bash[82222]: cluster 2024-03-07T03:21:22.078659+0000 mon.a (mon.0) 609 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2024-03-07T03:21:22.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:21:22 smithi195 bash[59134]: ts=2024-03-07T03:21:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:22 smithi195 bash[66542]: cluster 2024-03-07T03:21:21.147523+0000 mon.a (mon.0) 606 : cluster 0 osdmap e493: 8 total, 8 up, 8 in 2024-03-07T03:21:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:22 smithi195 bash[66542]: audit 2024-03-07T03:21:21.227302+0000 mon.c (mon.2) 18 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]: dispatch 2024-03-07T03:21:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:22 smithi195 bash[66542]: audit 2024-03-07T03:21:21.227903+0000 mon.a (mon.0) 607 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]: dispatch 2024-03-07T03:21:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:22 smithi195 bash[66542]: audit 2024-03-07T03:21:21.231441+0000 mon.a (mon.0) 608 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3bd6e44b-8860-4b5c-9ddf-eeb0eefb664d"}]': finished 2024-03-07T03:21:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:22 smithi195 bash[66542]: cluster 2024-03-07T03:21:22.078659+0000 mon.a (mon.0) 609 : cluster 0 osdmap e494: 8 total, 8 up, 8 in 2024-03-07T03:21:23.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.040 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.040 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.040 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.041 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.041 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.041 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:23 smithi148 systemd[1]: Stopping Ceph osd.1 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:21:23.042 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.042 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:21:22 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:23.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:23 smithi148 bash[79819]: cluster 2024-03-07T03:21:22.290623+0000 mgr.y (mgr.34101) 197 : cluster 0 pgmap v336: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 255 B/s wr, 10 op/s 2024-03-07T03:21:23.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:23 smithi148 bash[82222]: cluster 2024-03-07T03:21:22.290623+0000 mgr.y (mgr.34101) 197 : cluster 0 pgmap v336: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 255 B/s wr, 10 op/s 2024-03-07T03:21:23.540 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:23 smithi148 bash[39532]: debug 2024-03-07T03:21:23.137+0000 7f7d69b42700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:21:23.540 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:23 smithi148 bash[39532]: debug 2024-03-07T03:21:23.137+0000 7f7d69b42700 -1 osd.1 494 *** Got signal Terminated *** 2024-03-07T03:21:23.540 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:23 smithi148 bash[39532]: debug 2024-03-07T03:21:23.137+0000 7f7d69b42700 -1 osd.1 494 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:21:23.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:23 smithi195 bash[66542]: cluster 2024-03-07T03:21:22.290623+0000 mgr.y (mgr.34101) 197 : cluster 0 pgmap v336: 105 pgs: 105 active+clean; 583 KiB data, 421 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 255 B/s wr, 10 op/s 2024-03-07T03:21:24.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:24 smithi148 bash[82222]: cluster 2024-03-07T03:21:23.153541+0000 mon.a (mon.0) 610 : cluster 1 osd.1 marked itself down and dead 2024-03-07T03:21:24.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:24 smithi148 bash[82222]: cluster 2024-03-07T03:21:23.153585+0000 mon.a (mon.0) 611 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2024-03-07T03:21:24.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:24 smithi148 bash[79819]: debug 2024-03-07T03:21:24.145+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:24.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:24 smithi148 bash[79819]: cluster 2024-03-07T03:21:23.153541+0000 mon.a (mon.0) 610 : cluster 1 osd.1 marked itself down and dead 2024-03-07T03:21:24.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:24 smithi148 bash[79819]: cluster 2024-03-07T03:21:23.153585+0000 mon.a (mon.0) 611 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2024-03-07T03:21:24.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:24 smithi195 bash[66542]: cluster 2024-03-07T03:21:23.153541+0000 mon.a (mon.0) 610 : cluster 1 osd.1 marked itself down and dead 2024-03-07T03:21:24.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:24 smithi195 bash[66542]: cluster 2024-03-07T03:21:23.153585+0000 mon.a (mon.0) 611 : cluster 0 osdmap e495: 8 total, 8 up, 8 in 2024-03-07T03:21:25.476 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[93122]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:21:25.476 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[82222]: cluster 2024-03-07T03:21:24.150054+0000 mon.a (mon.0) 612 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:25.476 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[82222]: cluster 2024-03-07T03:21:24.150079+0000 mon.a (mon.0) 613 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:25.476 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[82222]: cluster 2024-03-07T03:21:24.159337+0000 mon.a (mon.0) 614 : cluster 0 osdmap e496: 8 total, 7 up, 8 in 2024-03-07T03:21:25.477 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[82222]: cluster 2024-03-07T03:21:24.291317+0000 mgr.y (mgr.34101) 198 : cluster 0 pgmap v339: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 439 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 9.7 KiB/s wr, 86 op/s 2024-03-07T03:21:25.477 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[79819]: cluster 2024-03-07T03:21:24.150054+0000 mon.a (mon.0) 612 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:25.477 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[79819]: cluster 2024-03-07T03:21:24.150079+0000 mon.a (mon.0) 613 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:25.477 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[79819]: cluster 2024-03-07T03:21:24.159337+0000 mon.a (mon.0) 614 : cluster 0 osdmap e496: 8 total, 7 up, 8 in 2024-03-07T03:21:25.477 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[79819]: cluster 2024-03-07T03:21:24.291317+0000 mgr.y (mgr.34101) 198 : cluster 0 pgmap v339: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 439 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 9.7 KiB/s wr, 86 op/s 2024-03-07T03:21:25.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:25 smithi195 bash[66542]: cluster 2024-03-07T03:21:24.150054+0000 mon.a (mon.0) 612 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:21:25.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:25 smithi195 bash[66542]: cluster 2024-03-07T03:21:24.150079+0000 mon.a (mon.0) 613 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:25.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:25 smithi195 bash[66542]: cluster 2024-03-07T03:21:24.159337+0000 mon.a (mon.0) 614 : cluster 0 osdmap e496: 8 total, 7 up, 8 in 2024-03-07T03:21:25.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:25 smithi195 bash[66542]: cluster 2024-03-07T03:21:24.291317+0000 mgr.y (mgr.34101) 198 : cluster 0 pgmap v339: 105 pgs: 12 stale+active+clean, 93 active+clean; 583 KiB data, 439 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 9.7 KiB/s wr, 86 op/s 2024-03-07T03:21:25.789 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:25 smithi148 bash[93206]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:21:26.147 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:21:26.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:26 smithi148 bash[79819]: cluster 2024-03-07T03:21:25.173589+0000 mon.a (mon.0) 615 : cluster 0 osdmap e497: 8 total, 7 up, 8 in 2024-03-07T03:21:26.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:26 smithi148 bash[82222]: cluster 2024-03-07T03:21:25.173589+0000 mon.a (mon.0) 615 : cluster 0 osdmap e497: 8 total, 7 up, 8 in 2024-03-07T03:21:26.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:26 smithi195 bash[66542]: cluster 2024-03-07T03:21:25.173589+0000 mon.a (mon.0) 615 : cluster 0 osdmap e497: 8 total, 7 up, 8 in 2024-03-07T03:21:27.425 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:21:27] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2024-03-07T03:21:27.426 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:27 smithi148 bash[79819]: cluster 2024-03-07T03:21:26.175404+0000 mon.a (mon.0) 616 : cluster 0 osdmap e498: 8 total, 7 up, 8 in 2024-03-07T03:21:27.426 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:27 smithi148 bash[79819]: cluster 2024-03-07T03:21:26.292108+0000 mgr.y (mgr.34101) 199 : cluster 0 pgmap v342: 105 pgs: 4 active+undersized+degraded, 11 stale+active+clean, 3 active+undersized, 87 active+clean; 583 KiB data, 439 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 10 KiB/s wr, 92 op/s; 8/339 objects degraded (2.360%) 2024-03-07T03:21:27.426 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:27 smithi148 bash[79819]: cluster 2024-03-07T03:21:27.075980+0000 mon.a (mon.0) 617 : cluster 0 osdmap e499: 8 total, 7 up, 8 in 2024-03-07T03:21:27.426 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:27 smithi148 bash[82222]: cluster 2024-03-07T03:21:26.175404+0000 mon.a (mon.0) 616 : cluster 0 osdmap e498: 8 total, 7 up, 8 in 2024-03-07T03:21:27.426 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:27 smithi148 bash[82222]: cluster 2024-03-07T03:21:26.292108+0000 mgr.y (mgr.34101) 199 : cluster 0 pgmap v342: 105 pgs: 4 active+undersized+degraded, 11 stale+active+clean, 3 active+undersized, 87 active+clean; 583 KiB data, 439 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 10 KiB/s wr, 92 op/s; 8/339 objects degraded (2.360%) 2024-03-07T03:21:27.426 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:27 smithi148 bash[82222]: cluster 2024-03-07T03:21:27.075980+0000 mon.a (mon.0) 617 : cluster 0 osdmap e499: 8 total, 7 up, 8 in 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:27 smithi195 bash[66542]: cluster 2024-03-07T03:21:26.175404+0000 mon.a (mon.0) 616 : cluster 0 osdmap e498: 8 total, 7 up, 8 in 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:27 smithi195 bash[66542]: cluster 2024-03-07T03:21:26.292108+0000 mgr.y (mgr.34101) 199 : cluster 0 pgmap v342: 105 pgs: 4 active+undersized+degraded, 11 stale+active+clean, 3 active+undersized, 87 active+clean; 583 KiB data, 439 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 10 KiB/s wr, 92 op/s; 8/339 objects degraded (2.360%) 2024-03-07T03:21:27.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:27 smithi195 bash[66542]: cluster 2024-03-07T03:21:27.075980+0000 mon.a (mon.0) 617 : cluster 0 osdmap e499: 8 total, 7 up, 8 in 2024-03-07T03:21:28.471 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-07T03:21:28.472 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: Stopped Ceph osd.1 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:21:28.472 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[79819]: cluster 2024-03-07T03:21:27.178409+0000 mon.a (mon.0) 618 : cluster 3 Health check failed: Degraded data redundancy: 8/339 objects degraded (2.360%), 4 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:28.472 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[79819]: audit 2024-03-07T03:21:27.313335+0000 mon.a (mon.0) 619 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:28.472 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[79819]: audit 2024-03-07T03:21:27.314469+0000 mon.a (mon.0) 620 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:28.472 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[79819]: cluster 2024-03-07T03:21:28.082384+0000 mon.a (mon.0) 621 : cluster 0 osdmap e500: 8 total, 7 up, 8 in 2024-03-07T03:21:28.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[82222]: cluster 2024-03-07T03:21:27.178409+0000 mon.a (mon.0) 618 : cluster 3 Health check failed: Degraded data redundancy: 8/339 objects degraded (2.360%), 4 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:28.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[82222]: audit 2024-03-07T03:21:27.313335+0000 mon.a (mon.0) 619 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:28.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[82222]: audit 2024-03-07T03:21:27.314469+0000 mon.a (mon.0) 620 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:28.473 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:28 smithi148 bash[82222]: cluster 2024-03-07T03:21:28.082384+0000 mon.a (mon.0) 621 : cluster 0 osdmap e500: 8 total, 7 up, 8 in 2024-03-07T03:21:28.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:28 smithi195 bash[66542]: cluster 2024-03-07T03:21:27.178409+0000 mon.a (mon.0) 618 : cluster 3 Health check failed: Degraded data redundancy: 8/339 objects degraded (2.360%), 4 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:28.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:28 smithi195 bash[66542]: audit 2024-03-07T03:21:27.313335+0000 mon.a (mon.0) 619 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:28.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:28 smithi195 bash[66542]: audit 2024-03-07T03:21:27.314469+0000 mon.a (mon.0) 620 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:28.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:28 smithi195 bash[66542]: cluster 2024-03-07T03:21:28.082384+0000 mon.a (mon.0) 621 : cluster 0 osdmap e500: 8 total, 7 up, 8 in 2024-03-07T03:21:28.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.790 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.790 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.790 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.791 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.791 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.791 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.791 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:21:28.792 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:28 smithi148 systemd[1]: Started Ceph osd.1 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:21:29.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[79819]: cluster 2024-03-07T03:21:28.293573+0000 mgr.y (mgr.34101) 200 : cluster 0 pgmap v345: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 9.5 KiB/s wr, 76 op/s; 47/381 objects degraded (12.336%) 2024-03-07T03:21:29.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[79819]: audit 2024-03-07T03:21:28.657277+0000 mon.a (mon.0) 622 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[79819]: audit 2024-03-07T03:21:28.663999+0000 mon.a (mon.0) 623 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[79819]: audit 2024-03-07T03:21:28.684640+0000 mon.a (mon.0) 624 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[79819]: cluster 2024-03-07T03:21:29.089608+0000 mon.a (mon.0) 625 : cluster 0 osdmap e501: 8 total, 7 up, 8 in 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[79819]: audit 2024-03-07T03:21:29.155180+0000 mon.a (mon.0) 626 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[79819]: audit 2024-03-07T03:21:29.160689+0000 mon.a (mon.0) 627 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[82222]: cluster 2024-03-07T03:21:28.293573+0000 mgr.y (mgr.34101) 200 : cluster 0 pgmap v345: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 9.5 KiB/s wr, 76 op/s; 47/381 objects degraded (12.336%) 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[82222]: audit 2024-03-07T03:21:28.657277+0000 mon.a (mon.0) 622 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[82222]: audit 2024-03-07T03:21:28.663999+0000 mon.a (mon.0) 623 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[82222]: audit 2024-03-07T03:21:28.684640+0000 mon.a (mon.0) 624 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[82222]: cluster 2024-03-07T03:21:29.089608+0000 mon.a (mon.0) 625 : cluster 0 osdmap e501: 8 total, 7 up, 8 in 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[82222]: audit 2024-03-07T03:21:29.155180+0000 mon.a (mon.0) 626 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:29 smithi148 bash[82222]: audit 2024-03-07T03:21:29.160689+0000 mon.a (mon.0) 627 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:29 smithi195 bash[66542]: cluster 2024-03-07T03:21:28.293573+0000 mgr.y (mgr.34101) 200 : cluster 0 pgmap v345: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 9.5 KiB/s wr, 76 op/s; 47/381 objects degraded (12.336%) 2024-03-07T03:21:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:29 smithi195 bash[66542]: audit 2024-03-07T03:21:28.657277+0000 mon.a (mon.0) 622 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:29 smithi195 bash[66542]: audit 2024-03-07T03:21:28.663999+0000 mon.a (mon.0) 623 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:29 smithi195 bash[66542]: audit 2024-03-07T03:21:28.684640+0000 mon.a (mon.0) 624 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:29 smithi195 bash[66542]: cluster 2024-03-07T03:21:29.089608+0000 mon.a (mon.0) 625 : cluster 0 osdmap e501: 8 total, 7 up, 8 in 2024-03-07T03:21:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:29 smithi195 bash[66542]: audit 2024-03-07T03:21:29.155180+0000 mon.a (mon.0) 626 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:29 smithi195 bash[66542]: audit 2024-03-07T03:21:29.160689+0000 mon.a (mon.0) 627 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:31.039 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:30 smithi148 bash[93440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-07T03:21:31.040 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:30 smithi148 bash[93440]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-07T03:21:31.040 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:30 smithi148 bash[93440]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-07T03:21:31.040 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:30 smithi148 bash[93440]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-07T03:21:31.040 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:30 smithi148 bash[93440]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-07T03:21:31.040 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:30 smithi148 bash[93440]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-07T03:21:31.040 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:30 smithi148 bash[93440]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-07T03:21:31.445 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:31 smithi148 bash[79819]: cluster 2024-03-07T03:21:30.086537+0000 mon.a (mon.0) 628 : cluster 0 osdmap e502: 8 total, 7 up, 8 in 2024-03-07T03:21:31.445 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:31 smithi148 bash[79819]: cluster 2024-03-07T03:21:30.294619+0000 mgr.y (mgr.34101) 201 : cluster 0 pgmap v348: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 11 KiB/s wr, 95 op/s; 51/402 objects degraded (12.687%) 2024-03-07T03:21:31.445 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:31 smithi148 bash[82222]: cluster 2024-03-07T03:21:30.086537+0000 mon.a (mon.0) 628 : cluster 0 osdmap e502: 8 total, 7 up, 8 in 2024-03-07T03:21:31.445 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:31 smithi148 bash[82222]: cluster 2024-03-07T03:21:30.294619+0000 mgr.y (mgr.34101) 201 : cluster 0 pgmap v348: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 11 KiB/s wr, 95 op/s; 51/402 objects degraded (12.687%) 2024-03-07T03:21:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:31 smithi195 bash[66542]: cluster 2024-03-07T03:21:30.086537+0000 mon.a (mon.0) 628 : cluster 0 osdmap e502: 8 total, 7 up, 8 in 2024-03-07T03:21:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:31 smithi195 bash[66542]: cluster 2024-03-07T03:21:30.294619+0000 mgr.y (mgr.34101) 201 : cluster 0 pgmap v348: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 11 KiB/s wr, 95 op/s; 51/402 objects degraded (12.687%) 2024-03-07T03:21:32.365 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:32 smithi148 bash[82222]: cluster 2024-03-07T03:21:31.100002+0000 mon.a (mon.0) 629 : cluster 0 osdmap e503: 8 total, 7 up, 8 in 2024-03-07T03:21:32.366 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:32 smithi148 bash[82222]: cluster 2024-03-07T03:21:32.081648+0000 mon.a (mon.0) 630 : cluster 0 osdmap e504: 8 total, 7 up, 8 in 2024-03-07T03:21:32.366 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:32 smithi148 bash[79819]: cluster 2024-03-07T03:21:31.100002+0000 mon.a (mon.0) 629 : cluster 0 osdmap e503: 8 total, 7 up, 8 in 2024-03-07T03:21:32.366 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:32 smithi148 bash[79819]: cluster 2024-03-07T03:21:32.081648+0000 mon.a (mon.0) 630 : cluster 0 osdmap e504: 8 total, 7 up, 8 in 2024-03-07T03:21:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:32 smithi195 bash[66542]: cluster 2024-03-07T03:21:31.100002+0000 mon.a (mon.0) 629 : cluster 0 osdmap e503: 8 total, 7 up, 8 in 2024-03-07T03:21:32.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:32 smithi195 bash[66542]: cluster 2024-03-07T03:21:32.081648+0000 mon.a (mon.0) 630 : cluster 0 osdmap e504: 8 total, 7 up, 8 in 2024-03-07T03:21:32.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:21:32 smithi195 bash[59134]: ts=2024-03-07T03:21:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:32.789 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:32 smithi148 bash[93712]: debug 2024-03-07T03:21:32.373+0000 7fc9f4cdc700 1 -- 172.21.15.148:0/3382794674 <== mon.0 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5619f4271680 con 0x5619f42acc00 2024-03-07T03:21:33.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:33 smithi148 bash[79819]: cluster 2024-03-07T03:21:32.295345+0000 mgr.y (mgr.34101) 202 : cluster 0 pgmap v351: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 2.5 KiB/s wr, 24 op/s; 51/402 objects degraded (12.687%) 2024-03-07T03:21:33.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:33 smithi148 bash[82222]: cluster 2024-03-07T03:21:32.295345+0000 mgr.y (mgr.34101) 202 : cluster 0 pgmap v351: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 2.5 KiB/s wr, 24 op/s; 51/402 objects degraded (12.687%) 2024-03-07T03:21:33.290 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:32 smithi148 bash[93712]: debug 2024-03-07T03:21:32.909+0000 7fc9fde747c0 -1 Falling back to public interface 2024-03-07T03:21:33.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:33 smithi195 bash[66542]: cluster 2024-03-07T03:21:32.295345+0000 mgr.y (mgr.34101) 202 : cluster 0 pgmap v351: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 583 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 15 KiB/s rd, 2.5 KiB/s wr, 24 op/s; 51/402 objects degraded (12.687%) 2024-03-07T03:21:34.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:34 smithi148 bash[79819]: cluster 2024-03-07T03:21:33.113184+0000 mon.a (mon.0) 631 : cluster 0 osdmap e505: 8 total, 7 up, 8 in 2024-03-07T03:21:34.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:34 smithi148 bash[82222]: cluster 2024-03-07T03:21:33.113184+0000 mon.a (mon.0) 631 : cluster 0 osdmap e505: 8 total, 7 up, 8 in 2024-03-07T03:21:34.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:34 smithi195 bash[66542]: cluster 2024-03-07T03:21:33.113184+0000 mon.a (mon.0) 631 : cluster 0 osdmap e505: 8 total, 7 up, 8 in 2024-03-07T03:21:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:35 smithi148 bash[79819]: cluster 2024-03-07T03:21:34.115740+0000 mon.a (mon.0) 632 : cluster 0 osdmap e506: 8 total, 7 up, 8 in 2024-03-07T03:21:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:35 smithi148 bash[79819]: cluster 2024-03-07T03:21:34.296248+0000 mgr.y (mgr.34101) 203 : cluster 0 pgmap v354: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 585 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 167 KiB/s rd, 21 KiB/s wr, 221 op/s; 71/498 objects degraded (14.257%) 2024-03-07T03:21:35.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:35 smithi148 bash[82222]: cluster 2024-03-07T03:21:34.115740+0000 mon.a (mon.0) 632 : cluster 0 osdmap e506: 8 total, 7 up, 8 in 2024-03-07T03:21:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:35 smithi148 bash[82222]: cluster 2024-03-07T03:21:34.296248+0000 mgr.y (mgr.34101) 203 : cluster 0 pgmap v354: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 585 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 167 KiB/s rd, 21 KiB/s wr, 221 op/s; 71/498 objects degraded (14.257%) 2024-03-07T03:21:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:35 smithi195 bash[66542]: cluster 2024-03-07T03:21:34.115740+0000 mon.a (mon.0) 632 : cluster 0 osdmap e506: 8 total, 7 up, 8 in 2024-03-07T03:21:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:35 smithi195 bash[66542]: cluster 2024-03-07T03:21:34.296248+0000 mgr.y (mgr.34101) 203 : cluster 0 pgmap v354: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 585 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 167 KiB/s rd, 21 KiB/s wr, 221 op/s; 71/498 objects degraded (14.257%) 2024-03-07T03:21:36.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:36 smithi148 bash[82222]: cluster 2024-03-07T03:21:35.119162+0000 mon.a (mon.0) 633 : cluster 3 Health check update: Degraded data redundancy: 71/498 objects degraded (14.257%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:36.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:36 smithi148 bash[82222]: cluster 2024-03-07T03:21:35.137292+0000 mon.a (mon.0) 634 : cluster 0 osdmap e507: 8 total, 7 up, 8 in 2024-03-07T03:21:36.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:36 smithi148 bash[82222]: audit 2024-03-07T03:21:35.980446+0000 mon.a (mon.0) 635 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:21:36.290 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:35 smithi148 bash[93712]: debug 2024-03-07T03:21:35.973+0000 7fc9fde747c0 -1 osd.1 495 log_to_monitors true 2024-03-07T03:21:36.290 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:21:36 smithi148 bash[93712]: debug 2024-03-07T03:21:36.185+0000 7fc9ef2cc700 -1 osd.1 495 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:21:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:36 smithi148 bash[79819]: cluster 2024-03-07T03:21:35.119162+0000 mon.a (mon.0) 633 : cluster 3 Health check update: Degraded data redundancy: 71/498 objects degraded (14.257%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:36 smithi148 bash[79819]: cluster 2024-03-07T03:21:35.137292+0000 mon.a (mon.0) 634 : cluster 0 osdmap e507: 8 total, 7 up, 8 in 2024-03-07T03:21:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:36 smithi148 bash[79819]: audit 2024-03-07T03:21:35.980446+0000 mon.a (mon.0) 635 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:21:36.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:36 smithi195 bash[66542]: cluster 2024-03-07T03:21:35.119162+0000 mon.a (mon.0) 633 : cluster 3 Health check update: Degraded data redundancy: 71/498 objects degraded (14.257%), 20 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:36.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:36 smithi195 bash[66542]: cluster 2024-03-07T03:21:35.137292+0000 mon.a (mon.0) 634 : cluster 0 osdmap e507: 8 total, 7 up, 8 in 2024-03-07T03:21:36.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:36 smithi195 bash[66542]: audit 2024-03-07T03:21:35.980446+0000 mon.a (mon.0) 635 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-07T03:21:37.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:21:37] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2024-03-07T03:21:37.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[79819]: debug 2024-03-07T03:21:37.137+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[79819]: audit 2024-03-07T03:21:36.139124+0000 mon.a (mon.0) 636 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[79819]: cluster 2024-03-07T03:21:36.147612+0000 mon.a (mon.0) 637 : cluster 0 osdmap e508: 8 total, 7 up, 8 in 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[79819]: audit 2024-03-07T03:21:36.148585+0000 mon.a (mon.0) 638 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[79819]: cluster 2024-03-07T03:21:36.297161+0000 mgr.y (mgr.34101) 204 : cluster 0 pgmap v357: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 585 KiB data, 420 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 22 KiB/s wr, 241 op/s; 72/507 objects degraded (14.201%) 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[82222]: audit 2024-03-07T03:21:36.139124+0000 mon.a (mon.0) 636 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[82222]: cluster 2024-03-07T03:21:36.147612+0000 mon.a (mon.0) 637 : cluster 0 osdmap e508: 8 total, 7 up, 8 in 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[82222]: audit 2024-03-07T03:21:36.148585+0000 mon.a (mon.0) 638 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:21:37.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:37 smithi148 bash[82222]: cluster 2024-03-07T03:21:36.297161+0000 mgr.y (mgr.34101) 204 : cluster 0 pgmap v357: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 585 KiB data, 420 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 22 KiB/s wr, 241 op/s; 72/507 objects degraded (14.201%) 2024-03-07T03:21:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:37 smithi195 bash[66542]: audit 2024-03-07T03:21:36.139124+0000 mon.a (mon.0) 636 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-07T03:21:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:37 smithi195 bash[66542]: cluster 2024-03-07T03:21:36.147612+0000 mon.a (mon.0) 637 : cluster 0 osdmap e508: 8 total, 7 up, 8 in 2024-03-07T03:21:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:37 smithi195 bash[66542]: audit 2024-03-07T03:21:36.148585+0000 mon.a (mon.0) 638 : audit 1 from='osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:21:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:37 smithi195 bash[66542]: cluster 2024-03-07T03:21:36.297161+0000 mgr.y (mgr.34101) 204 : cluster 0 pgmap v357: 105 pgs: 20 active+undersized+degraded, 29 active+undersized, 56 active+clean; 585 KiB data, 420 MiB used, 715 GiB / 715 GiB avail; 180 KiB/s rd, 22 KiB/s wr, 241 op/s; 72/507 objects degraded (14.201%) 2024-03-07T03:21:38.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[79819]: cluster 2024-03-07T03:21:37.140354+0000 mon.a (mon.0) 639 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[79819]: cluster 2024-03-07T03:21:37.140385+0000 mon.a (mon.0) 640 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[79819]: cluster 2024-03-07T03:21:37.153481+0000 mon.a (mon.0) 641 : cluster 1 osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856] boot 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[79819]: cluster 2024-03-07T03:21:37.153575+0000 mon.a (mon.0) 642 : cluster 0 osdmap e509: 8 total, 8 up, 8 in 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[79819]: audit 2024-03-07T03:21:37.154806+0000 mon.a (mon.0) 643 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[79819]: audit 2024-03-07T03:21:37.780944+0000 mon.a (mon.0) 644 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[79819]: audit 2024-03-07T03:21:37.788676+0000 mon.a (mon.0) 645 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[82222]: cluster 2024-03-07T03:21:37.140354+0000 mon.a (mon.0) 639 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[82222]: cluster 2024-03-07T03:21:37.140385+0000 mon.a (mon.0) 640 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[82222]: cluster 2024-03-07T03:21:37.153481+0000 mon.a (mon.0) 641 : cluster 1 osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856] boot 2024-03-07T03:21:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[82222]: cluster 2024-03-07T03:21:37.153575+0000 mon.a (mon.0) 642 : cluster 0 osdmap e509: 8 total, 8 up, 8 in 2024-03-07T03:21:38.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[82222]: audit 2024-03-07T03:21:37.154806+0000 mon.a (mon.0) 643 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:21:38.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[82222]: audit 2024-03-07T03:21:37.780944+0000 mon.a (mon.0) 644 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:38.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:38 smithi148 bash[82222]: audit 2024-03-07T03:21:37.788676+0000 mon.a (mon.0) 645 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:38 smithi195 bash[66542]: cluster 2024-03-07T03:21:37.140354+0000 mon.a (mon.0) 639 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:21:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:38 smithi195 bash[66542]: cluster 2024-03-07T03:21:37.140385+0000 mon.a (mon.0) 640 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:21:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:38 smithi195 bash[66542]: cluster 2024-03-07T03:21:37.153481+0000 mon.a (mon.0) 641 : cluster 1 osd.1 [v2:172.21.15.148:6810/2696710856,v1:172.21.15.148:6811/2696710856] boot 2024-03-07T03:21:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:38 smithi195 bash[66542]: cluster 2024-03-07T03:21:37.153575+0000 mon.a (mon.0) 642 : cluster 0 osdmap e509: 8 total, 8 up, 8 in 2024-03-07T03:21:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:38 smithi195 bash[66542]: audit 2024-03-07T03:21:37.154806+0000 mon.a (mon.0) 643 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-07T03:21:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:38 smithi195 bash[66542]: audit 2024-03-07T03:21:37.780944+0000 mon.a (mon.0) 644 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:38 smithi195 bash[66542]: audit 2024-03-07T03:21:37.788676+0000 mon.a (mon.0) 645 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:39.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[79819]: cluster 2024-03-07T03:21:38.158312+0000 mon.a (mon.0) 646 : cluster 0 osdmap e510: 8 total, 8 up, 8 in 2024-03-07T03:21:39.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[79819]: cluster 2024-03-07T03:21:38.297901+0000 mgr.y (mgr.34101) 205 : cluster 0 pgmap v360: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 11 active+undersized, 79 active+clean; 584 KiB data, 420 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 7.2 KiB/s wr, 87 op/s; 41/480 objects degraded (8.542%); 47 B/s, 29 keys/s, 2 objects/s recovering 2024-03-07T03:21:39.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[79819]: audit 2024-03-07T03:21:38.828450+0000 mon.a (mon.0) 647 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:39.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[79819]: audit 2024-03-07T03:21:38.836320+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[82222]: cluster 2024-03-07T03:21:38.158312+0000 mon.a (mon.0) 646 : cluster 0 osdmap e510: 8 total, 8 up, 8 in 2024-03-07T03:21:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[82222]: cluster 2024-03-07T03:21:38.297901+0000 mgr.y (mgr.34101) 205 : cluster 0 pgmap v360: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 11 active+undersized, 79 active+clean; 584 KiB data, 420 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 7.2 KiB/s wr, 87 op/s; 41/480 objects degraded (8.542%); 47 B/s, 29 keys/s, 2 objects/s recovering 2024-03-07T03:21:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[82222]: audit 2024-03-07T03:21:38.828450+0000 mon.a (mon.0) 647 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:39 smithi148 bash[82222]: audit 2024-03-07T03:21:38.836320+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:39 smithi195 bash[66542]: cluster 2024-03-07T03:21:38.158312+0000 mon.a (mon.0) 646 : cluster 0 osdmap e510: 8 total, 8 up, 8 in 2024-03-07T03:21:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:39 smithi195 bash[66542]: cluster 2024-03-07T03:21:38.297901+0000 mgr.y (mgr.34101) 205 : cluster 0 pgmap v360: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 7 active+undersized+degraded, 5 active+recovery_wait+degraded, 11 active+undersized, 79 active+clean; 584 KiB data, 420 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 7.2 KiB/s wr, 87 op/s; 41/480 objects degraded (8.542%); 47 B/s, 29 keys/s, 2 objects/s recovering 2024-03-07T03:21:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:39 smithi195 bash[66542]: audit 2024-03-07T03:21:38.828450+0000 mon.a (mon.0) 647 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:39 smithi195 bash[66542]: audit 2024-03-07T03:21:38.836320+0000 mon.a (mon.0) 648 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:40 smithi148 bash[79819]: cluster 2024-03-07T03:21:39.193055+0000 mon.a (mon.0) 649 : cluster 0 osdmap e511: 8 total, 8 up, 8 in 2024-03-07T03:21:40.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:40 smithi148 bash[82222]: cluster 2024-03-07T03:21:39.193055+0000 mon.a (mon.0) 649 : cluster 0 osdmap e511: 8 total, 8 up, 8 in 2024-03-07T03:21:40.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:40 smithi195 bash[66542]: cluster 2024-03-07T03:21:39.193055+0000 mon.a (mon.0) 649 : cluster 0 osdmap e511: 8 total, 8 up, 8 in 2024-03-07T03:21:41.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:41 smithi148 bash[79819]: cluster 2024-03-07T03:21:40.210739+0000 mon.a (mon.0) 650 : cluster 0 osdmap e512: 8 total, 8 up, 8 in 2024-03-07T03:21:41.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:41 smithi148 bash[79819]: cluster 2024-03-07T03:21:40.298603+0000 mgr.y (mgr.34101) 206 : cluster 0 pgmap v363: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 3 active+undersized+degraded, 5 active+recovery_wait+degraded, 9 active+undersized, 85 active+clean; 584 KiB data, 425 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.7 KiB/s wr, 69 op/s; 28/468 objects degraded (5.983%); 87 B/s, 31 keys/s, 3 objects/s recovering 2024-03-07T03:21:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:41 smithi148 bash[82222]: cluster 2024-03-07T03:21:40.210739+0000 mon.a (mon.0) 650 : cluster 0 osdmap e512: 8 total, 8 up, 8 in 2024-03-07T03:21:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:41 smithi148 bash[82222]: cluster 2024-03-07T03:21:40.298603+0000 mgr.y (mgr.34101) 206 : cluster 0 pgmap v363: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 3 active+undersized+degraded, 5 active+recovery_wait+degraded, 9 active+undersized, 85 active+clean; 584 KiB data, 425 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.7 KiB/s wr, 69 op/s; 28/468 objects degraded (5.983%); 87 B/s, 31 keys/s, 3 objects/s recovering 2024-03-07T03:21:41.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:41 smithi195 bash[66542]: cluster 2024-03-07T03:21:40.210739+0000 mon.a (mon.0) 650 : cluster 0 osdmap e512: 8 total, 8 up, 8 in 2024-03-07T03:21:41.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:41 smithi195 bash[66542]: cluster 2024-03-07T03:21:40.298603+0000 mgr.y (mgr.34101) 206 : cluster 0 pgmap v363: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 3 active+undersized+degraded, 5 active+recovery_wait+degraded, 9 active+undersized, 85 active+clean; 584 KiB data, 425 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 5.7 KiB/s wr, 69 op/s; 28/468 objects degraded (5.983%); 87 B/s, 31 keys/s, 3 objects/s recovering 2024-03-07T03:21:42.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:42 smithi148 bash[79819]: cluster 2024-03-07T03:21:41.199368+0000 mon.a (mon.0) 651 : cluster 3 Health check update: Degraded data redundancy: 28/468 objects degraded (5.983%), 8 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:42.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:42 smithi148 bash[79819]: cluster 2024-03-07T03:21:41.218164+0000 mon.a (mon.0) 652 : cluster 0 osdmap e513: 8 total, 8 up, 8 in 2024-03-07T03:21:42.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:42 smithi148 bash[82222]: cluster 2024-03-07T03:21:41.199368+0000 mon.a (mon.0) 651 : cluster 3 Health check update: Degraded data redundancy: 28/468 objects degraded (5.983%), 8 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:42.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:42 smithi148 bash[82222]: cluster 2024-03-07T03:21:41.218164+0000 mon.a (mon.0) 652 : cluster 0 osdmap e513: 8 total, 8 up, 8 in 2024-03-07T03:21:42.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:42 smithi195 bash[66542]: cluster 2024-03-07T03:21:41.199368+0000 mon.a (mon.0) 651 : cluster 3 Health check update: Degraded data redundancy: 28/468 objects degraded (5.983%), 8 pgs degraded (PG_DEGRADED) 2024-03-07T03:21:42.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:42 smithi195 bash[66542]: cluster 2024-03-07T03:21:41.218164+0000 mon.a (mon.0) 652 : cluster 0 osdmap e513: 8 total, 8 up, 8 in 2024-03-07T03:21:42.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:21:42 smithi195 bash[59134]: ts=2024-03-07T03:21:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:43.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[79819]: cluster 2024-03-07T03:21:42.220050+0000 mon.a (mon.0) 653 : cluster 0 osdmap e514: 8 total, 8 up, 8 in 2024-03-07T03:21:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[79819]: cluster 2024-03-07T03:21:42.299564+0000 mgr.y (mgr.34101) 207 : cluster 0 pgmap v366: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 5 active+recovery_wait+degraded, 97 active+clean; 584 KiB data, 425 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 B/s wr, 2 op/s; 11/465 objects degraded (2.366%); 81 B/s, 7 keys/s, 3 objects/s recovering 2024-03-07T03:21:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[79819]: audit 2024-03-07T03:21:42.318150+0000 mon.a (mon.0) 654 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[79819]: audit 2024-03-07T03:21:42.320667+0000 mon.a (mon.0) 655 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:43.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[82222]: cluster 2024-03-07T03:21:42.220050+0000 mon.a (mon.0) 653 : cluster 0 osdmap e514: 8 total, 8 up, 8 in 2024-03-07T03:21:43.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[82222]: cluster 2024-03-07T03:21:42.299564+0000 mgr.y (mgr.34101) 207 : cluster 0 pgmap v366: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 5 active+recovery_wait+degraded, 97 active+clean; 584 KiB data, 425 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 B/s wr, 2 op/s; 11/465 objects degraded (2.366%); 81 B/s, 7 keys/s, 3 objects/s recovering 2024-03-07T03:21:43.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[82222]: audit 2024-03-07T03:21:42.318150+0000 mon.a (mon.0) 654 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:43.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:43 smithi148 bash[82222]: audit 2024-03-07T03:21:42.320667+0000 mon.a (mon.0) 655 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:43 smithi195 bash[66542]: cluster 2024-03-07T03:21:42.220050+0000 mon.a (mon.0) 653 : cluster 0 osdmap e514: 8 total, 8 up, 8 in 2024-03-07T03:21:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:43 smithi195 bash[66542]: cluster 2024-03-07T03:21:42.299564+0000 mgr.y (mgr.34101) 207 : cluster 0 pgmap v366: 105 pgs: 2 active+recovery_wait, 1 active+clean+snaptrim_wait, 5 active+recovery_wait+degraded, 97 active+clean; 584 KiB data, 425 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 0 B/s wr, 2 op/s; 11/465 objects degraded (2.366%); 81 B/s, 7 keys/s, 3 objects/s recovering 2024-03-07T03:21:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:43 smithi195 bash[66542]: audit 2024-03-07T03:21:42.318150+0000 mon.a (mon.0) 654 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:43.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:43 smithi195 bash[66542]: audit 2024-03-07T03:21:42.320667+0000 mon.a (mon.0) 655 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:44 smithi148 bash[79819]: cluster 2024-03-07T03:21:43.322255+0000 mon.a (mon.0) 656 : cluster 0 osdmap e515: 8 total, 8 up, 8 in 2024-03-07T03:21:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:44 smithi148 bash[82222]: cluster 2024-03-07T03:21:43.322255+0000 mon.a (mon.0) 656 : cluster 0 osdmap e515: 8 total, 8 up, 8 in 2024-03-07T03:21:44.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:44 smithi195 bash[66542]: cluster 2024-03-07T03:21:43.322255+0000 mon.a (mon.0) 656 : cluster 0 osdmap e515: 8 total, 8 up, 8 in 2024-03-07T03:21:45.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:45 smithi148 bash[79819]: cluster 2024-03-07T03:21:44.300352+0000 mgr.y (mgr.34101) 208 : cluster 0 pgmap v368: 105 pgs: 105 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 5.1 KiB/s wr, 87 op/s; 172 B/s, 7 keys/s, 5 objects/s recovering 2024-03-07T03:21:45.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:45 smithi148 bash[79819]: cluster 2024-03-07T03:21:44.315957+0000 mon.a (mon.0) 657 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/465 objects degraded (2.366%), 5 pgs degraded) 2024-03-07T03:21:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:45 smithi148 bash[79819]: cluster 2024-03-07T03:21:44.338089+0000 mon.a (mon.0) 658 : cluster 0 osdmap e516: 8 total, 8 up, 8 in 2024-03-07T03:21:45.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:45 smithi148 bash[82222]: cluster 2024-03-07T03:21:44.300352+0000 mgr.y (mgr.34101) 208 : cluster 0 pgmap v368: 105 pgs: 105 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 5.1 KiB/s wr, 87 op/s; 172 B/s, 7 keys/s, 5 objects/s recovering 2024-03-07T03:21:45.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:45 smithi148 bash[82222]: cluster 2024-03-07T03:21:44.315957+0000 mon.a (mon.0) 657 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/465 objects degraded (2.366%), 5 pgs degraded) 2024-03-07T03:21:45.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:45 smithi148 bash[82222]: cluster 2024-03-07T03:21:44.338089+0000 mon.a (mon.0) 658 : cluster 0 osdmap e516: 8 total, 8 up, 8 in 2024-03-07T03:21:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:45 smithi195 bash[66542]: cluster 2024-03-07T03:21:44.300352+0000 mgr.y (mgr.34101) 208 : cluster 0 pgmap v368: 105 pgs: 105 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 5.1 KiB/s wr, 87 op/s; 172 B/s, 7 keys/s, 5 objects/s recovering 2024-03-07T03:21:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:45 smithi195 bash[66542]: cluster 2024-03-07T03:21:44.315957+0000 mon.a (mon.0) 657 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 11/465 objects degraded (2.366%), 5 pgs degraded) 2024-03-07T03:21:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:45 smithi195 bash[66542]: cluster 2024-03-07T03:21:44.338089+0000 mon.a (mon.0) 658 : cluster 0 osdmap e516: 8 total, 8 up, 8 in 2024-03-07T03:21:46.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:46 smithi148 bash[79819]: cluster 2024-03-07T03:21:45.343614+0000 mon.a (mon.0) 659 : cluster 0 osdmap e517: 8 total, 8 up, 8 in 2024-03-07T03:21:46.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:46 smithi148 bash[82222]: cluster 2024-03-07T03:21:45.343614+0000 mon.a (mon.0) 659 : cluster 0 osdmap e517: 8 total, 8 up, 8 in 2024-03-07T03:21:46.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:46 smithi195 bash[66542]: cluster 2024-03-07T03:21:45.343614+0000 mon.a (mon.0) 659 : cluster 0 osdmap e517: 8 total, 8 up, 8 in 2024-03-07T03:21:47.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:21:47] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2024-03-07T03:21:47.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:47 smithi148 bash[79819]: cluster 2024-03-07T03:21:46.301175+0000 mgr.y (mgr.34101) 209 : cluster 0 pgmap v371: 105 pgs: 105 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 11 KiB/s wr, 197 op/s; 92 B/s, 2 objects/s recovering 2024-03-07T03:21:47.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:47 smithi148 bash[79819]: cluster 2024-03-07T03:21:46.348532+0000 mon.a (mon.0) 660 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2024-03-07T03:21:47.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:47 smithi148 bash[79819]: cluster 2024-03-07T03:21:47.085303+0000 mon.a (mon.0) 661 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2024-03-07T03:21:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:47 smithi148 bash[82222]: cluster 2024-03-07T03:21:46.301175+0000 mgr.y (mgr.34101) 209 : cluster 0 pgmap v371: 105 pgs: 105 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 11 KiB/s wr, 197 op/s; 92 B/s, 2 objects/s recovering 2024-03-07T03:21:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:47 smithi148 bash[82222]: cluster 2024-03-07T03:21:46.348532+0000 mon.a (mon.0) 660 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2024-03-07T03:21:47.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:47 smithi148 bash[82222]: cluster 2024-03-07T03:21:47.085303+0000 mon.a (mon.0) 661 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2024-03-07T03:21:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:47 smithi195 bash[66542]: cluster 2024-03-07T03:21:46.301175+0000 mgr.y (mgr.34101) 209 : cluster 0 pgmap v371: 105 pgs: 105 active+clean; 584 KiB data, 429 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 11 KiB/s wr, 197 op/s; 92 B/s, 2 objects/s recovering 2024-03-07T03:21:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:47 smithi195 bash[66542]: cluster 2024-03-07T03:21:46.348532+0000 mon.a (mon.0) 660 : cluster 0 osdmap e518: 8 total, 8 up, 8 in 2024-03-07T03:21:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:47 smithi195 bash[66542]: cluster 2024-03-07T03:21:47.085303+0000 mon.a (mon.0) 661 : cluster 0 osdmap e519: 8 total, 8 up, 8 in 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[79819]: cluster 2024-03-07T03:21:48.085573+0000 mon.a (mon.0) 662 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[79819]: cluster 2024-03-07T03:21:48.302392+0000 mgr.y (mgr.34101) 210 : cluster 0 pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 12 KiB/s wr, 219 op/s 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[79819]: audit 2024-03-07T03:21:49.002936+0000 mon.a (mon.0) 663 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[79819]: audit 2024-03-07T03:21:49.013689+0000 mon.a (mon.0) 664 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[79819]: audit 2024-03-07T03:21:49.015993+0000 mon.a (mon.0) 665 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[79819]: audit 2024-03-07T03:21:49.017758+0000 mon.a (mon.0) 666 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[79819]: audit 2024-03-07T03:21:49.028085+0000 mon.a (mon.0) 667 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[82222]: cluster 2024-03-07T03:21:48.085573+0000 mon.a (mon.0) 662 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2024-03-07T03:21:49.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[82222]: cluster 2024-03-07T03:21:48.302392+0000 mgr.y (mgr.34101) 210 : cluster 0 pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 12 KiB/s wr, 219 op/s 2024-03-07T03:21:49.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[82222]: audit 2024-03-07T03:21:49.002936+0000 mon.a (mon.0) 663 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[82222]: audit 2024-03-07T03:21:49.013689+0000 mon.a (mon.0) 664 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[82222]: audit 2024-03-07T03:21:49.015993+0000 mon.a (mon.0) 665 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:49.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[82222]: audit 2024-03-07T03:21:49.017758+0000 mon.a (mon.0) 666 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:49.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:49 smithi148 bash[82222]: audit 2024-03-07T03:21:49.028085+0000 mon.a (mon.0) 667 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:49 smithi195 bash[66542]: cluster 2024-03-07T03:21:48.085573+0000 mon.a (mon.0) 662 : cluster 0 osdmap e520: 8 total, 8 up, 8 in 2024-03-07T03:21:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:49 smithi195 bash[66542]: cluster 2024-03-07T03:21:48.302392+0000 mgr.y (mgr.34101) 210 : cluster 0 pgmap v375: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 156 KiB/s rd, 12 KiB/s wr, 219 op/s 2024-03-07T03:21:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:49 smithi195 bash[66542]: audit 2024-03-07T03:21:49.002936+0000 mon.a (mon.0) 663 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:49 smithi195 bash[66542]: audit 2024-03-07T03:21:49.013689+0000 mon.a (mon.0) 664 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:49 smithi195 bash[66542]: audit 2024-03-07T03:21:49.015993+0000 mon.a (mon.0) 665 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:49 smithi195 bash[66542]: audit 2024-03-07T03:21:49.017758+0000 mon.a (mon.0) 666 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:49 smithi195 bash[66542]: audit 2024-03-07T03:21:49.028085+0000 mon.a (mon.0) 667 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:50.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:50 smithi148 bash[79819]: cluster 2024-03-07T03:21:49.101625+0000 mon.a (mon.0) 668 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2024-03-07T03:21:50.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:50 smithi148 bash[79819]: audit 2024-03-07T03:21:49.147966+0000 mon.a (mon.0) 669 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:50.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:50 smithi148 bash[82222]: cluster 2024-03-07T03:21:49.101625+0000 mon.a (mon.0) 668 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2024-03-07T03:21:50.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:50 smithi148 bash[82222]: audit 2024-03-07T03:21:49.147966+0000 mon.a (mon.0) 669 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:50.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:50 smithi195 bash[66542]: cluster 2024-03-07T03:21:49.101625+0000 mon.a (mon.0) 668 : cluster 0 osdmap e521: 8 total, 8 up, 8 in 2024-03-07T03:21:50.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:50 smithi195 bash[66542]: audit 2024-03-07T03:21:49.147966+0000 mon.a (mon.0) 669 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:21:51.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:51 smithi148 bash[79819]: cluster 2024-03-07T03:21:50.101134+0000 mon.a (mon.0) 670 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2024-03-07T03:21:51.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:51 smithi148 bash[79819]: cluster 2024-03-07T03:21:50.303240+0000 mgr.y (mgr.34101) 211 : cluster 0 pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.6 KiB/s wr, 108 op/s 2024-03-07T03:21:51.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:51 smithi148 bash[82222]: cluster 2024-03-07T03:21:50.101134+0000 mon.a (mon.0) 670 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2024-03-07T03:21:51.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:51 smithi148 bash[82222]: cluster 2024-03-07T03:21:50.303240+0000 mgr.y (mgr.34101) 211 : cluster 0 pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.6 KiB/s wr, 108 op/s 2024-03-07T03:21:51.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:51 smithi195 bash[66542]: cluster 2024-03-07T03:21:50.101134+0000 mon.a (mon.0) 670 : cluster 0 osdmap e522: 8 total, 8 up, 8 in 2024-03-07T03:21:51.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:51 smithi195 bash[66542]: cluster 2024-03-07T03:21:50.303240+0000 mgr.y (mgr.34101) 211 : cluster 0 pgmap v378: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.6 KiB/s wr, 108 op/s 2024-03-07T03:21:52.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:52 smithi148 bash[82222]: cluster 2024-03-07T03:21:51.122204+0000 mon.a (mon.0) 671 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2024-03-07T03:21:52.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:52 smithi148 bash[82222]: cluster 2024-03-07T03:21:52.090649+0000 mon.a (mon.0) 672 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2024-03-07T03:21:52.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:52 smithi148 bash[79819]: cluster 2024-03-07T03:21:51.122204+0000 mon.a (mon.0) 671 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2024-03-07T03:21:52.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:52 smithi148 bash[79819]: cluster 2024-03-07T03:21:52.090649+0000 mon.a (mon.0) 672 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2024-03-07T03:21:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:52 smithi195 bash[66542]: cluster 2024-03-07T03:21:51.122204+0000 mon.a (mon.0) 671 : cluster 0 osdmap e523: 8 total, 8 up, 8 in 2024-03-07T03:21:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:52 smithi195 bash[66542]: cluster 2024-03-07T03:21:52.090649+0000 mon.a (mon.0) 672 : cluster 0 osdmap e524: 8 total, 8 up, 8 in 2024-03-07T03:21:52.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:21:52 smithi195 bash[59134]: ts=2024-03-07T03:21:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:21:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:53 smithi148 bash[79819]: cluster 2024-03-07T03:21:52.303970+0000 mgr.y (mgr.34101) 212 : cluster 0 pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 255 B/s wr, 16 op/s 2024-03-07T03:21:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:53 smithi148 bash[79819]: audit 2024-03-07T03:21:52.429830+0000 mon.a (mon.0) 673 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:53.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:53 smithi148 bash[79819]: cluster 2024-03-07T03:21:53.134170+0000 mon.a (mon.0) 674 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2024-03-07T03:21:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:53 smithi148 bash[82222]: cluster 2024-03-07T03:21:52.303970+0000 mgr.y (mgr.34101) 212 : cluster 0 pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 255 B/s wr, 16 op/s 2024-03-07T03:21:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:53 smithi148 bash[82222]: audit 2024-03-07T03:21:52.429830+0000 mon.a (mon.0) 673 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:53 smithi148 bash[82222]: cluster 2024-03-07T03:21:53.134170+0000 mon.a (mon.0) 674 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2024-03-07T03:21:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:53 smithi195 bash[66542]: cluster 2024-03-07T03:21:52.303970+0000 mgr.y (mgr.34101) 212 : cluster 0 pgmap v381: 105 pgs: 105 active+clean; 583 KiB data, 434 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 255 B/s wr, 16 op/s 2024-03-07T03:21:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:53 smithi195 bash[66542]: audit 2024-03-07T03:21:52.429830+0000 mon.a (mon.0) 673 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:53 smithi195 bash[66542]: cluster 2024-03-07T03:21:53.134170+0000 mon.a (mon.0) 674 : cluster 0 osdmap e525: 8 total, 8 up, 8 in 2024-03-07T03:21:54.240 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestMirroring.AioSnapshotCreate (33010 ms) 2024-03-07T03:21:55.252 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 51 tests from TestMirroring (96896 ms total) 2024-03-07T03:21:55.252 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:21:55.252 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 34 tests from TestInternal 2024-03-07T03:21:55.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[79819]: cluster 2024-03-07T03:21:54.138325+0000 mon.a (mon.0) 675 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2024-03-07T03:21:55.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[79819]: audit 2024-03-07T03:21:54.234684+0000 mon.c (mon.2) 19 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]: dispatch 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[79819]: audit 2024-03-07T03:21:54.235390+0000 mon.a (mon.0) 676 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]: dispatch 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[79819]: audit 2024-03-07T03:21:54.241380+0000 mon.a (mon.0) 677 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]': finished 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[79819]: cluster 2024-03-07T03:21:54.304735+0000 mgr.y (mgr.34101) 213 : cluster 0 pgmap v384: 105 pgs: 105 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 7.7 KiB/s wr, 125 op/s 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[82222]: cluster 2024-03-07T03:21:54.138325+0000 mon.a (mon.0) 675 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[82222]: audit 2024-03-07T03:21:54.234684+0000 mon.c (mon.2) 19 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]: dispatch 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[82222]: audit 2024-03-07T03:21:54.235390+0000 mon.a (mon.0) 676 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]: dispatch 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[82222]: audit 2024-03-07T03:21:54.241380+0000 mon.a (mon.0) 677 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]': finished 2024-03-07T03:21:55.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:55 smithi148 bash[82222]: cluster 2024-03-07T03:21:54.304735+0000 mgr.y (mgr.34101) 213 : cluster 0 pgmap v384: 105 pgs: 105 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 7.7 KiB/s wr, 125 op/s 2024-03-07T03:21:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:55 smithi195 bash[66542]: cluster 2024-03-07T03:21:54.138325+0000 mon.a (mon.0) 675 : cluster 0 osdmap e526: 8 total, 8 up, 8 in 2024-03-07T03:21:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:55 smithi195 bash[66542]: audit 2024-03-07T03:21:54.234684+0000 mon.c (mon.2) 19 : audit 1 from='client.? 172.21.15.148:0/1326944891' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]: dispatch 2024-03-07T03:21:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:55 smithi195 bash[66542]: audit 2024-03-07T03:21:54.235390+0000 mon.a (mon.0) 676 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]: dispatch 2024-03-07T03:21:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:55 smithi195 bash[66542]: audit 2024-03-07T03:21:54.241380+0000 mon.a (mon.0) 677 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/243b2c89-64de-45be-b360-4ee1481814f9"}]': finished 2024-03-07T03:21:55.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:55 smithi195 bash[66542]: cluster 2024-03-07T03:21:54.304735+0000 mgr.y (mgr.34101) 213 : cluster 0 pgmap v384: 105 pgs: 105 active+clean; 583 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 7.7 KiB/s wr, 125 op/s 2024-03-07T03:21:56.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[79819]: audit 2024-03-07T03:21:55.195208+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[79819]: audit 2024-03-07T03:21:55.208834+0000 mon.a (mon.0) 679 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[79819]: cluster 2024-03-07T03:21:55.258329+0000 mon.a (mon.0) 680 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[79819]: audit 2024-03-07T03:21:55.889530+0000 mon.a (mon.0) 681 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[79819]: audit 2024-03-07T03:21:55.891622+0000 mon.a (mon.0) 682 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[79819]: audit 2024-03-07T03:21:55.904022+0000 mon.a (mon.0) 683 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[82222]: audit 2024-03-07T03:21:55.195208+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[82222]: audit 2024-03-07T03:21:55.208834+0000 mon.a (mon.0) 679 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[82222]: cluster 2024-03-07T03:21:55.258329+0000 mon.a (mon.0) 680 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2024-03-07T03:21:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[82222]: audit 2024-03-07T03:21:55.889530+0000 mon.a (mon.0) 681 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:56.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[82222]: audit 2024-03-07T03:21:55.891622+0000 mon.a (mon.0) 682 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:56.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:56 smithi148 bash[82222]: audit 2024-03-07T03:21:55.904022+0000 mon.a (mon.0) 683 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:56 smithi195 bash[66542]: audit 2024-03-07T03:21:55.195208+0000 mon.a (mon.0) 678 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:56 smithi195 bash[66542]: audit 2024-03-07T03:21:55.208834+0000 mon.a (mon.0) 679 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:56 smithi195 bash[66542]: cluster 2024-03-07T03:21:55.258329+0000 mon.a (mon.0) 680 : cluster 0 osdmap e527: 8 total, 8 up, 8 in 2024-03-07T03:21:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:56 smithi195 bash[66542]: audit 2024-03-07T03:21:55.889530+0000 mon.a (mon.0) 681 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:21:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:56 smithi195 bash[66542]: audit 2024-03-07T03:21:55.891622+0000 mon.a (mon.0) 682 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:21:56.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:56 smithi195 bash[66542]: audit 2024-03-07T03:21:55.904022+0000 mon.a (mon.0) 683 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:57.257 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.OpenByID 2024-03-07T03:21:57.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:21:57] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[79819]: cluster 2024-03-07T03:21:56.259539+0000 mon.a (mon.0) 684 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[79819]: audit 2024-03-07T03:21:56.263115+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.148:0/4213020569' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[79819]: audit 2024-03-07T03:21:56.265096+0000 mon.a (mon.0) 685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[79819]: cluster 2024-03-07T03:21:56.305437+0000 mgr.y (mgr.34101) 214 : cluster 0 pgmap v387: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[82222]: cluster 2024-03-07T03:21:56.259539+0000 mon.a (mon.0) 684 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[82222]: audit 2024-03-07T03:21:56.263115+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.148:0/4213020569' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[82222]: audit 2024-03-07T03:21:56.265096+0000 mon.a (mon.0) 685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:57.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:57 smithi148 bash[82222]: cluster 2024-03-07T03:21:56.305437+0000 mgr.y (mgr.34101) 214 : cluster 0 pgmap v387: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:57 smithi195 bash[66542]: cluster 2024-03-07T03:21:56.259539+0000 mon.a (mon.0) 684 : cluster 0 osdmap e528: 8 total, 8 up, 8 in 2024-03-07T03:21:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:57 smithi195 bash[66542]: audit 2024-03-07T03:21:56.263115+0000 mon.c (mon.2) 20 : audit 1 from='client.? 172.21.15.148:0/4213020569' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:57 smithi195 bash[66542]: audit 2024-03-07T03:21:56.265096+0000 mon.a (mon.0) 685 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:21:57.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:57 smithi195 bash[66542]: cluster 2024-03-07T03:21:56.305437+0000 mgr.y (mgr.34101) 214 : cluster 0 pgmap v387: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:58.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[79819]: cluster 2024-03-07T03:21:57.253354+0000 mon.a (mon.0) 686 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[79819]: audit 2024-03-07T03:21:57.262680+0000 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[79819]: cluster 2024-03-07T03:21:57.270979+0000 mon.a (mon.0) 688 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[79819]: audit 2024-03-07T03:21:57.315294+0000 mon.a (mon.0) 689 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[79819]: audit 2024-03-07T03:21:57.316623+0000 mon.a (mon.0) 690 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[82222]: cluster 2024-03-07T03:21:57.253354+0000 mon.a (mon.0) 686 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[82222]: audit 2024-03-07T03:21:57.262680+0000 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[82222]: cluster 2024-03-07T03:21:57.270979+0000 mon.a (mon.0) 688 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2024-03-07T03:21:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[82222]: audit 2024-03-07T03:21:57.315294+0000 mon.a (mon.0) 689 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:58.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:58 smithi148 bash[82222]: audit 2024-03-07T03:21:57.316623+0000 mon.a (mon.0) 690 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:58 smithi195 bash[66542]: cluster 2024-03-07T03:21:57.253354+0000 mon.a (mon.0) 686 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:21:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:58 smithi195 bash[66542]: audit 2024-03-07T03:21:57.262680+0000 mon.a (mon.0) 687 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:21:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:58 smithi195 bash[66542]: cluster 2024-03-07T03:21:57.270979+0000 mon.a (mon.0) 688 : cluster 0 osdmap e529: 8 total, 8 up, 8 in 2024-03-07T03:21:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:58 smithi195 bash[66542]: audit 2024-03-07T03:21:57.315294+0000 mon.a (mon.0) 689 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:21:58.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:58 smithi195 bash[66542]: audit 2024-03-07T03:21:57.316623+0000 mon.a (mon.0) 690 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:21:59.349 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.OpenByID (2092 ms) 2024-03-07T03:21:59.349 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-03-07T03:21:59.394 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.OpenSnapDNE (46 ms) 2024-03-07T03:21:59.395 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-03-07T03:21:59.453 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (58 ms) 2024-03-07T03:21:59.453 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-03-07T03:21:59.514 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.ResizeLocksImage (62 ms) 2024-03-07T03:21:59.514 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-03-07T03:21:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:59 smithi148 bash[79819]: cluster 2024-03-07T03:21:58.284567+0000 mon.a (mon.0) 691 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2024-03-07T03:21:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:21:59 smithi148 bash[79819]: cluster 2024-03-07T03:21:58.306114+0000 mgr.y (mgr.34101) 215 : cluster 0 pgmap v390: 105 pgs: 9 unknown, 96 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:59.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:59 smithi148 bash[82222]: cluster 2024-03-07T03:21:58.284567+0000 mon.a (mon.0) 691 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2024-03-07T03:21:59.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:21:59 smithi148 bash[82222]: cluster 2024-03-07T03:21:58.306114+0000 mgr.y (mgr.34101) 215 : cluster 0 pgmap v390: 105 pgs: 9 unknown, 96 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:59.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:59 smithi195 bash[66542]: cluster 2024-03-07T03:21:58.284567+0000 mon.a (mon.0) 691 : cluster 0 osdmap e530: 8 total, 8 up, 8 in 2024-03-07T03:21:59.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:21:59 smithi195 bash[66542]: cluster 2024-03-07T03:21:58.306114+0000 mgr.y (mgr.34101) 215 : cluster 0 pgmap v390: 105 pgs: 9 unknown, 96 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:21:59.566 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (51 ms) 2024-03-07T03:21:59.566 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-03-07T03:22:00.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:00 smithi148 bash[79819]: cluster 2024-03-07T03:21:59.275990+0000 mon.a (mon.0) 692 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:00.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:00 smithi148 bash[79819]: cluster 2024-03-07T03:21:59.294216+0000 mon.a (mon.0) 693 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2024-03-07T03:22:00.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:00 smithi148 bash[82222]: cluster 2024-03-07T03:21:59.275990+0000 mon.a (mon.0) 692 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:00.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:00 smithi148 bash[82222]: cluster 2024-03-07T03:21:59.294216+0000 mon.a (mon.0) 693 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2024-03-07T03:22:00.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:00 smithi195 bash[66542]: cluster 2024-03-07T03:21:59.275990+0000 mon.a (mon.0) 692 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:00.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:00 smithi195 bash[66542]: cluster 2024-03-07T03:21:59.294216+0000 mon.a (mon.0) 693 : cluster 0 osdmap e531: 8 total, 8 up, 8 in 2024-03-07T03:22:01.327 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1761 ms) 2024-03-07T03:22:01.327 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-03-07T03:22:01.390 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (63 ms) 2024-03-07T03:22:01.390 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-03-07T03:22:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:01 smithi195 bash[66542]: cluster 2024-03-07T03:22:00.302491+0000 mon.a (mon.0) 694 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2024-03-07T03:22:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:01 smithi195 bash[66542]: cluster 2024-03-07T03:22:00.306599+0000 mgr.y (mgr.34101) 216 : cluster 0 pgmap v393: 105 pgs: 3 unknown, 102 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 1.2 KiB/s wr, 25 op/s 2024-03-07T03:22:01.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:01 smithi148 bash[79819]: cluster 2024-03-07T03:22:00.302491+0000 mon.a (mon.0) 694 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2024-03-07T03:22:01.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:01 smithi148 bash[79819]: cluster 2024-03-07T03:22:00.306599+0000 mgr.y (mgr.34101) 216 : cluster 0 pgmap v393: 105 pgs: 3 unknown, 102 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 1.2 KiB/s wr, 25 op/s 2024-03-07T03:22:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:01 smithi148 bash[82222]: cluster 2024-03-07T03:22:00.302491+0000 mon.a (mon.0) 694 : cluster 0 osdmap e532: 8 total, 8 up, 8 in 2024-03-07T03:22:01.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:01 smithi148 bash[82222]: cluster 2024-03-07T03:22:00.306599+0000 mgr.y (mgr.34101) 216 : cluster 0 pgmap v393: 105 pgs: 3 unknown, 102 active+clean; 582 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 1.2 KiB/s wr, 25 op/s 2024-03-07T03:22:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:02 smithi195 bash[66542]: cluster 2024-03-07T03:22:01.321358+0000 mon.a (mon.0) 695 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2024-03-07T03:22:02.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:22:02 smithi195 bash[59134]: ts=2024-03-07T03:22:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:02.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:02 smithi148 bash[79819]: cluster 2024-03-07T03:22:01.321358+0000 mon.a (mon.0) 695 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2024-03-07T03:22:02.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:02 smithi148 bash[82222]: cluster 2024-03-07T03:22:01.321358+0000 mon.a (mon.0) 695 : cluster 0 osdmap e533: 8 total, 8 up, 8 in 2024-03-07T03:22:03.391 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (2001 ms) 2024-03-07T03:22:03.391 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-03-07T03:22:03.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:03 smithi195 bash[66542]: cluster 2024-03-07T03:22:02.307195+0000 mgr.y (mgr.34101) 217 : cluster 0 pgmap v395: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 34 op/s 2024-03-07T03:22:03.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:03 smithi195 bash[66542]: cluster 2024-03-07T03:22:02.327938+0000 mon.a (mon.0) 696 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2024-03-07T03:22:03.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:03 smithi148 bash[79819]: cluster 2024-03-07T03:22:02.307195+0000 mgr.y (mgr.34101) 217 : cluster 0 pgmap v395: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 34 op/s 2024-03-07T03:22:03.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:03 smithi148 bash[79819]: cluster 2024-03-07T03:22:02.327938+0000 mon.a (mon.0) 696 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2024-03-07T03:22:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:03 smithi148 bash[82222]: cluster 2024-03-07T03:22:02.307195+0000 mgr.y (mgr.34101) 217 : cluster 0 pgmap v395: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 3.2 KiB/s wr, 34 op/s 2024-03-07T03:22:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:03 smithi148 bash[82222]: cluster 2024-03-07T03:22:02.327938+0000 mon.a (mon.0) 696 : cluster 0 osdmap e534: 8 total, 8 up, 8 in 2024-03-07T03:22:04.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:04 smithi148 bash[79819]: cluster 2024-03-07T03:22:03.371822+0000 mon.a (mon.0) 697 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2024-03-07T03:22:04.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:04 smithi148 bash[82222]: cluster 2024-03-07T03:22:03.371822+0000 mon.a (mon.0) 697 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2024-03-07T03:22:04.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:04 smithi195 bash[66542]: cluster 2024-03-07T03:22:03.371822+0000 mon.a (mon.0) 697 : cluster 0 osdmap e535: 8 total, 8 up, 8 in 2024-03-07T03:22:05.414 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (2024 ms) 2024-03-07T03:22:05.414 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-03-07T03:22:05.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:05 smithi148 bash[79819]: cluster 2024-03-07T03:22:04.307868+0000 mgr.y (mgr.34101) 218 : cluster 0 pgmap v398: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 124 op/s 2024-03-07T03:22:05.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:05 smithi148 bash[79819]: cluster 2024-03-07T03:22:04.383876+0000 mon.a (mon.0) 698 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2024-03-07T03:22:05.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:05 smithi148 bash[82222]: cluster 2024-03-07T03:22:04.307868+0000 mgr.y (mgr.34101) 218 : cluster 0 pgmap v398: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 124 op/s 2024-03-07T03:22:05.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:05 smithi148 bash[82222]: cluster 2024-03-07T03:22:04.383876+0000 mon.a (mon.0) 698 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2024-03-07T03:22:05.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:05 smithi195 bash[66542]: cluster 2024-03-07T03:22:04.307868+0000 mgr.y (mgr.34101) 218 : cluster 0 pgmap v398: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s rd, 10 KiB/s wr, 124 op/s 2024-03-07T03:22:05.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:05 smithi195 bash[66542]: cluster 2024-03-07T03:22:04.383876+0000 mon.a (mon.0) 698 : cluster 0 osdmap e536: 8 total, 8 up, 8 in 2024-03-07T03:22:06.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:06 smithi148 bash[79819]: cluster 2024-03-07T03:22:05.385265+0000 mon.a (mon.0) 699 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2024-03-07T03:22:06.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:06 smithi148 bash[82222]: cluster 2024-03-07T03:22:05.385265+0000 mon.a (mon.0) 699 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2024-03-07T03:22:06.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:06 smithi195 bash[66542]: cluster 2024-03-07T03:22:05.385265+0000 mon.a (mon.0) 699 : cluster 0 osdmap e537: 8 total, 8 up, 8 in 2024-03-07T03:22:07.128 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapSetReleasesLock (1713 ms) 2024-03-07T03:22:07.128 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-03-07T03:22:07.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:22:07] "GET /metrics HTTP/1.1" 200 36464 "" "Prometheus/2.43.0" 2024-03-07T03:22:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:07 smithi148 bash[79819]: cluster 2024-03-07T03:22:06.308810+0000 mgr.y (mgr.34101) 219 : cluster 0 pgmap v401: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 10 KiB/s wr, 131 op/s 2024-03-07T03:22:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:07 smithi148 bash[79819]: cluster 2024-03-07T03:22:06.404788+0000 mon.a (mon.0) 700 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2024-03-07T03:22:07.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:07 smithi148 bash[79819]: cluster 2024-03-07T03:22:07.098131+0000 mon.a (mon.0) 701 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2024-03-07T03:22:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:07 smithi148 bash[82222]: cluster 2024-03-07T03:22:06.308810+0000 mgr.y (mgr.34101) 219 : cluster 0 pgmap v401: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 10 KiB/s wr, 131 op/s 2024-03-07T03:22:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:07 smithi148 bash[82222]: cluster 2024-03-07T03:22:06.404788+0000 mon.a (mon.0) 700 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2024-03-07T03:22:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:07 smithi148 bash[82222]: cluster 2024-03-07T03:22:07.098131+0000 mon.a (mon.0) 701 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2024-03-07T03:22:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:07 smithi195 bash[66542]: cluster 2024-03-07T03:22:06.308810+0000 mgr.y (mgr.34101) 219 : cluster 0 pgmap v401: 105 pgs: 105 active+clean; 582 KiB data, 447 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s rd, 10 KiB/s wr, 131 op/s 2024-03-07T03:22:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:07 smithi195 bash[66542]: cluster 2024-03-07T03:22:06.404788+0000 mon.a (mon.0) 700 : cluster 0 osdmap e538: 8 total, 8 up, 8 in 2024-03-07T03:22:07.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:07 smithi195 bash[66542]: cluster 2024-03-07T03:22:07.098131+0000 mon.a (mon.0) 701 : cluster 0 osdmap e539: 8 total, 8 up, 8 in 2024-03-07T03:22:08.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:08 smithi148 bash[82222]: audit 2024-03-07T03:22:07.453398+0000 mon.a (mon.0) 702 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:08.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:08 smithi148 bash[82222]: cluster 2024-03-07T03:22:08.110426+0000 mon.a (mon.0) 703 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2024-03-07T03:22:08.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:08 smithi148 bash[79819]: audit 2024-03-07T03:22:07.453398+0000 mon.a (mon.0) 702 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:08.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:08 smithi148 bash[79819]: cluster 2024-03-07T03:22:08.110426+0000 mon.a (mon.0) 703 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2024-03-07T03:22:08.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:08 smithi195 bash[66542]: audit 2024-03-07T03:22:07.453398+0000 mon.a (mon.0) 702 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:08.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:08 smithi195 bash[66542]: cluster 2024-03-07T03:22:08.110426+0000 mon.a (mon.0) 703 : cluster 0 osdmap e540: 8 total, 8 up, 8 in 2024-03-07T03:22:09.155 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.FlattenLocksImage (2027 ms) 2024-03-07T03:22:09.155 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-03-07T03:22:09.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:09 smithi148 bash[82222]: cluster 2024-03-07T03:22:08.310108+0000 mgr.y (mgr.34101) 220 : cluster 0 pgmap v405: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 8.6 KiB/s wr, 134 op/s 2024-03-07T03:22:09.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:09 smithi148 bash[82222]: cluster 2024-03-07T03:22:09.124461+0000 mon.a (mon.0) 704 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2024-03-07T03:22:09.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:09 smithi148 bash[79819]: cluster 2024-03-07T03:22:08.310108+0000 mgr.y (mgr.34101) 220 : cluster 0 pgmap v405: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 8.6 KiB/s wr, 134 op/s 2024-03-07T03:22:09.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:09 smithi148 bash[79819]: cluster 2024-03-07T03:22:09.124461+0000 mon.a (mon.0) 704 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2024-03-07T03:22:09.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:09 smithi195 bash[66542]: cluster 2024-03-07T03:22:08.310108+0000 mgr.y (mgr.34101) 220 : cluster 0 pgmap v405: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s rd, 8.6 KiB/s wr, 134 op/s 2024-03-07T03:22:09.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:09 smithi195 bash[66542]: cluster 2024-03-07T03:22:09.124461+0000 mon.a (mon.0) 704 : cluster 0 osdmap e541: 8 total, 8 up, 8 in 2024-03-07T03:22:11.152 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (1997 ms) 2024-03-07T03:22:11.152 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-03-07T03:22:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:11 smithi148 bash[79819]: cluster 2024-03-07T03:22:10.124882+0000 mon.a (mon.0) 705 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2024-03-07T03:22:11.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:11 smithi148 bash[79819]: cluster 2024-03-07T03:22:10.310965+0000 mgr.y (mgr.34101) 221 : cluster 0 pgmap v408: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 8.9 KiB/s wr, 171 op/s 2024-03-07T03:22:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:11 smithi148 bash[82222]: cluster 2024-03-07T03:22:10.124882+0000 mon.a (mon.0) 705 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2024-03-07T03:22:11.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:11 smithi148 bash[82222]: cluster 2024-03-07T03:22:10.310965+0000 mgr.y (mgr.34101) 221 : cluster 0 pgmap v408: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 8.9 KiB/s wr, 171 op/s 2024-03-07T03:22:11.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:11 smithi195 bash[66542]: cluster 2024-03-07T03:22:10.124882+0000 mon.a (mon.0) 705 : cluster 0 osdmap e542: 8 total, 8 up, 8 in 2024-03-07T03:22:11.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:11 smithi195 bash[66542]: cluster 2024-03-07T03:22:10.310965+0000 mgr.y (mgr.34101) 221 : cluster 0 pgmap v408: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 126 KiB/s rd, 8.9 KiB/s wr, 171 op/s 2024-03-07T03:22:12.093 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (942 ms) 2024-03-07T03:22:12.094 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-03-07T03:22:12.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[79819]: cluster 2024-03-07T03:22:11.137234+0000 mon.a (mon.0) 706 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2024-03-07T03:22:12.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[79819]: audit 2024-03-07T03:22:11.249316+0000 mon.c (mon.2) 21 : audit 1 from='client.? 172.21.15.148:0/2190593811' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]: dispatch 2024-03-07T03:22:12.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[79819]: audit 2024-03-07T03:22:11.249749+0000 mon.a (mon.0) 707 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]: dispatch 2024-03-07T03:22:12.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[79819]: audit 2024-03-07T03:22:12.093447+0000 mon.a (mon.0) 708 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]': finished 2024-03-07T03:22:12.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[79819]: cluster 2024-03-07T03:22:12.098932+0000 mon.a (mon.0) 709 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2024-03-07T03:22:12.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[82222]: cluster 2024-03-07T03:22:11.137234+0000 mon.a (mon.0) 706 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2024-03-07T03:22:12.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[82222]: audit 2024-03-07T03:22:11.249316+0000 mon.c (mon.2) 21 : audit 1 from='client.? 172.21.15.148:0/2190593811' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]: dispatch 2024-03-07T03:22:12.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[82222]: audit 2024-03-07T03:22:11.249749+0000 mon.a (mon.0) 707 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]: dispatch 2024-03-07T03:22:12.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[82222]: audit 2024-03-07T03:22:12.093447+0000 mon.a (mon.0) 708 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]': finished 2024-03-07T03:22:12.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:12 smithi148 bash[82222]: cluster 2024-03-07T03:22:12.098932+0000 mon.a (mon.0) 709 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2024-03-07T03:22:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:22:12 smithi195 bash[59134]: ts=2024-03-07T03:22:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:12 smithi195 bash[66542]: cluster 2024-03-07T03:22:11.137234+0000 mon.a (mon.0) 706 : cluster 0 osdmap e543: 8 total, 8 up, 8 in 2024-03-07T03:22:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:12 smithi195 bash[66542]: audit 2024-03-07T03:22:11.249316+0000 mon.c (mon.2) 21 : audit 1 from='client.? 172.21.15.148:0/2190593811' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]: dispatch 2024-03-07T03:22:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:12 smithi195 bash[66542]: audit 2024-03-07T03:22:11.249749+0000 mon.a (mon.0) 707 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]: dispatch 2024-03-07T03:22:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:12 smithi195 bash[66542]: audit 2024-03-07T03:22:12.093447+0000 mon.a (mon.0) 708 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2190593811"}]': finished 2024-03-07T03:22:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:12 smithi195 bash[66542]: cluster 2024-03-07T03:22:12.098932+0000 mon.a (mon.0) 709 : cluster 0 osdmap e544: 8 total, 8 up, 8 in 2024-03-07T03:22:13.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[79819]: audit 2024-03-07T03:22:12.164239+0000 mon.a (mon.0) 710 : audit 1 from='client.? 172.21.15.148:0/3198375452' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/3198375452"}]: dispatch 2024-03-07T03:22:13.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[79819]: cluster 2024-03-07T03:22:12.311677+0000 mgr.y (mgr.34101) 222 : cluster 0 pgmap v411: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.5 KiB/s wr, 60 op/s 2024-03-07T03:22:13.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[79819]: audit 2024-03-07T03:22:12.318871+0000 mon.a (mon.0) 711 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:13.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[79819]: audit 2024-03-07T03:22:12.321190+0000 mon.a (mon.0) 712 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:13.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[82222]: audit 2024-03-07T03:22:12.164239+0000 mon.a (mon.0) 710 : audit 1 from='client.? 172.21.15.148:0/3198375452' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/3198375452"}]: dispatch 2024-03-07T03:22:13.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[82222]: cluster 2024-03-07T03:22:12.311677+0000 mgr.y (mgr.34101) 222 : cluster 0 pgmap v411: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.5 KiB/s wr, 60 op/s 2024-03-07T03:22:13.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[82222]: audit 2024-03-07T03:22:12.318871+0000 mon.a (mon.0) 711 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:13.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:13 smithi148 bash[82222]: audit 2024-03-07T03:22:12.321190+0000 mon.a (mon.0) 712 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:13.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:13 smithi195 bash[66542]: audit 2024-03-07T03:22:12.164239+0000 mon.a (mon.0) 710 : audit 1 from='client.? 172.21.15.148:0/3198375452' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/3198375452"}]: dispatch 2024-03-07T03:22:13.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:13 smithi195 bash[66542]: cluster 2024-03-07T03:22:12.311677+0000 mgr.y (mgr.34101) 222 : cluster 0 pgmap v411: 105 pgs: 105 active+clean; 583 KiB data, 436 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.5 KiB/s wr, 60 op/s 2024-03-07T03:22:13.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:13 smithi195 bash[66542]: audit 2024-03-07T03:22:12.318871+0000 mon.a (mon.0) 711 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:13.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:13 smithi195 bash[66542]: audit 2024-03-07T03:22:12.321190+0000 mon.a (mon.0) 712 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:14.141 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (2047 ms) 2024-03-07T03:22:14.141 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-03-07T03:22:14.200 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.AioWriteRequestsLock (60 ms) 2024-03-07T03:22:14.200 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-03-07T03:22:14.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (58 ms) 2024-03-07T03:22:14.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-03-07T03:22:14.371 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.CancelAsyncResize (113 ms) 2024-03-07T03:22:14.372 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.MultipleResize 2024-03-07T03:22:14.513 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.MultipleResize (141 ms) 2024-03-07T03:22:14.513 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.Metadata 2024-03-07T03:22:14.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:14 smithi148 bash[79819]: audit 2024-03-07T03:22:13.142725+0000 mon.a (mon.0) 713 : audit 1 from='client.? 172.21.15.148:0/3198375452' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/3198375452"}]': finished 2024-03-07T03:22:14.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:14 smithi148 bash[79819]: cluster 2024-03-07T03:22:13.150059+0000 mon.a (mon.0) 714 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2024-03-07T03:22:14.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:14 smithi148 bash[82222]: audit 2024-03-07T03:22:13.142725+0000 mon.a (mon.0) 713 : audit 1 from='client.? 172.21.15.148:0/3198375452' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/3198375452"}]': finished 2024-03-07T03:22:14.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:14 smithi148 bash[82222]: cluster 2024-03-07T03:22:13.150059+0000 mon.a (mon.0) 714 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2024-03-07T03:22:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:14 smithi195 bash[66542]: audit 2024-03-07T03:22:13.142725+0000 mon.a (mon.0) 713 : audit 1 from='client.? 172.21.15.148:0/3198375452' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/3198375452"}]': finished 2024-03-07T03:22:14.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:14 smithi195 bash[66542]: cluster 2024-03-07T03:22:13.150059+0000 mon.a (mon.0) 714 : cluster 0 osdmap e545: 8 total, 8 up, 8 in 2024-03-07T03:22:14.661 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.Metadata (148 ms) 2024-03-07T03:22:14.661 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-03-07T03:22:14.750 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.MetadataConfApply (90 ms) 2024-03-07T03:22:14.750 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-03-07T03:22:15.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:15 smithi148 bash[79819]: cluster 2024-03-07T03:22:14.313065+0000 mgr.y (mgr.34101) 223 : cluster 0 pgmap v413: 105 pgs: 105 active+clean; 583 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 7.6 KiB/s wr, 157 op/s 2024-03-07T03:22:15.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:15 smithi148 bash[82222]: cluster 2024-03-07T03:22:14.313065+0000 mgr.y (mgr.34101) 223 : cluster 0 pgmap v413: 105 pgs: 105 active+clean; 583 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 7.6 KiB/s wr, 157 op/s 2024-03-07T03:22:15.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:15 smithi195 bash[66542]: cluster 2024-03-07T03:22:14.313065+0000 mgr.y (mgr.34101) 223 : cluster 0 pgmap v413: 105 pgs: 105 active+clean; 583 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 7.6 KiB/s wr, 157 op/s 2024-03-07T03:22:16.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:16 smithi148 bash[79819]: cluster 2024-03-07T03:22:15.168384+0000 mon.a (mon.0) 715 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2024-03-07T03:22:16.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:16 smithi148 bash[82222]: cluster 2024-03-07T03:22:15.168384+0000 mon.a (mon.0) 715 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2024-03-07T03:22:16.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:16 smithi195 bash[66542]: cluster 2024-03-07T03:22:15.168384+0000 mon.a (mon.0) 715 : cluster 0 osdmap e546: 8 total, 8 up, 8 in 2024-03-07T03:22:17.325 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapshotCopyup (2575 ms) 2024-03-07T03:22:17.325 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-03-07T03:22:17.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:22:17] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2024-03-07T03:22:17.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:17 smithi148 bash[79819]: cluster 2024-03-07T03:22:16.173446+0000 mon.a (mon.0) 716 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2024-03-07T03:22:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:17 smithi148 bash[79819]: cluster 2024-03-07T03:22:16.313950+0000 mgr.y (mgr.34101) 224 : cluster 0 pgmap v416: 105 pgs: 105 active+clean; 583 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 15 KiB/s wr, 207 op/s 2024-03-07T03:22:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:17 smithi148 bash[79819]: cluster 2024-03-07T03:22:17.104915+0000 mon.a (mon.0) 717 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2024-03-07T03:22:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:17 smithi148 bash[82222]: cluster 2024-03-07T03:22:16.173446+0000 mon.a (mon.0) 716 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2024-03-07T03:22:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:17 smithi148 bash[82222]: cluster 2024-03-07T03:22:16.313950+0000 mgr.y (mgr.34101) 224 : cluster 0 pgmap v416: 105 pgs: 105 active+clean; 583 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 15 KiB/s wr, 207 op/s 2024-03-07T03:22:17.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:17 smithi148 bash[82222]: cluster 2024-03-07T03:22:17.104915+0000 mon.a (mon.0) 717 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2024-03-07T03:22:17.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:17 smithi195 bash[66542]: cluster 2024-03-07T03:22:16.173446+0000 mon.a (mon.0) 716 : cluster 0 osdmap e547: 8 total, 8 up, 8 in 2024-03-07T03:22:17.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:17 smithi195 bash[66542]: cluster 2024-03-07T03:22:16.313950+0000 mgr.y (mgr.34101) 224 : cluster 0 pgmap v416: 105 pgs: 105 active+clean; 583 KiB data, 437 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 15 KiB/s wr, 207 op/s 2024-03-07T03:22:17.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:17 smithi195 bash[66542]: cluster 2024-03-07T03:22:17.104915+0000 mon.a (mon.0) 717 : cluster 0 osdmap e548: 8 total, 8 up, 8 in 2024-03-07T03:22:19.252 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1926 ms) 2024-03-07T03:22:19.252 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-03-07T03:22:19.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:19 smithi148 bash[79819]: cluster 2024-03-07T03:22:18.187701+0000 mon.a (mon.0) 718 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2024-03-07T03:22:19.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:19 smithi148 bash[79819]: cluster 2024-03-07T03:22:18.315270+0000 mgr.y (mgr.34101) 225 : cluster 0 pgmap v419: 105 pgs: 105 active+clean; 585 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 297 KiB/s rd, 32 KiB/s wr, 417 op/s 2024-03-07T03:22:19.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:19 smithi148 bash[82222]: cluster 2024-03-07T03:22:18.187701+0000 mon.a (mon.0) 718 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2024-03-07T03:22:19.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:19 smithi148 bash[82222]: cluster 2024-03-07T03:22:18.315270+0000 mgr.y (mgr.34101) 225 : cluster 0 pgmap v419: 105 pgs: 105 active+clean; 585 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 297 KiB/s rd, 32 KiB/s wr, 417 op/s 2024-03-07T03:22:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:19 smithi195 bash[66542]: cluster 2024-03-07T03:22:18.187701+0000 mon.a (mon.0) 718 : cluster 0 osdmap e549: 8 total, 8 up, 8 in 2024-03-07T03:22:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:19 smithi195 bash[66542]: cluster 2024-03-07T03:22:18.315270+0000 mgr.y (mgr.34101) 225 : cluster 0 pgmap v419: 105 pgs: 105 active+clean; 585 KiB data, 441 MiB used, 715 GiB / 715 GiB avail; 297 KiB/s rd, 32 KiB/s wr, 417 op/s 2024-03-07T03:22:20.251 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (1000 ms) 2024-03-07T03:22:20.251 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-03-07T03:22:20.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:20 smithi148 bash[79819]: cluster 2024-03-07T03:22:19.197599+0000 mon.a (mon.0) 719 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2024-03-07T03:22:20.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:20 smithi148 bash[82222]: cluster 2024-03-07T03:22:19.197599+0000 mon.a (mon.0) 719 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2024-03-07T03:22:20.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:20 smithi195 bash[66542]: cluster 2024-03-07T03:22:19.197599+0000 mon.a (mon.0) 719 : cluster 0 osdmap e550: 8 total, 8 up, 8 in 2024-03-07T03:22:21.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:21 smithi148 bash[82222]: cluster 2024-03-07T03:22:20.203452+0000 mon.a (mon.0) 720 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2024-03-07T03:22:21.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:21 smithi148 bash[82222]: cluster 2024-03-07T03:22:20.316267+0000 mgr.y (mgr.34101) 226 : cluster 0 pgmap v422: 105 pgs: 105 active+clean; 585 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 283 KiB/s rd, 25 KiB/s wr, 385 op/s 2024-03-07T03:22:21.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:21 smithi148 bash[79819]: cluster 2024-03-07T03:22:20.203452+0000 mon.a (mon.0) 720 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2024-03-07T03:22:21.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:21 smithi148 bash[79819]: cluster 2024-03-07T03:22:20.316267+0000 mgr.y (mgr.34101) 226 : cluster 0 pgmap v422: 105 pgs: 105 active+clean; 585 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 283 KiB/s rd, 25 KiB/s wr, 385 op/s 2024-03-07T03:22:21.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:21 smithi195 bash[66542]: cluster 2024-03-07T03:22:20.203452+0000 mon.a (mon.0) 720 : cluster 0 osdmap e551: 8 total, 8 up, 8 in 2024-03-07T03:22:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:21 smithi195 bash[66542]: cluster 2024-03-07T03:22:20.316267+0000 mgr.y (mgr.34101) 226 : cluster 0 pgmap v422: 105 pgs: 105 active+clean; 585 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 283 KiB/s rd, 25 KiB/s wr, 385 op/s 2024-03-07T03:22:22.211 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.ResizeCopyup (1960 ms) 2024-03-07T03:22:22.212 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-03-07T03:22:22.232 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_internal.cc:1056: Skipped 2024-03-07T03:22:22.233 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:22:22.233 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestInternal.DiscardCopyup (21 ms) 2024-03-07T03:22:22.233 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.ImageOptions 2024-03-07T03:22:22.253 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.ImageOptions (21 ms) 2024-03-07T03:22:22.253 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-03-07T03:22:22.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:22 smithi148 bash[79819]: cluster 2024-03-07T03:22:21.214624+0000 mon.a (mon.0) 721 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2024-03-07T03:22:22.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:22 smithi148 bash[79819]: cluster 2024-03-07T03:22:22.103536+0000 mon.a (mon.0) 722 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2024-03-07T03:22:22.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:22 smithi148 bash[82222]: cluster 2024-03-07T03:22:21.214624+0000 mon.a (mon.0) 721 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2024-03-07T03:22:22.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:22 smithi148 bash[82222]: cluster 2024-03-07T03:22:22.103536+0000 mon.a (mon.0) 722 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2024-03-07T03:22:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:22 smithi195 bash[66542]: cluster 2024-03-07T03:22:21.214624+0000 mon.a (mon.0) 721 : cluster 0 osdmap e552: 8 total, 8 up, 8 in 2024-03-07T03:22:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:22 smithi195 bash[66542]: cluster 2024-03-07T03:22:22.103536+0000 mon.a (mon.0) 722 : cluster 0 osdmap e553: 8 total, 8 up, 8 in 2024-03-07T03:22:22.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:22:22 smithi195 bash[59134]: ts=2024-03-07T03:22:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:23.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:23 smithi148 bash[79819]: cluster 2024-03-07T03:22:22.316929+0000 mgr.y (mgr.34101) 227 : cluster 0 pgmap v425: 105 pgs: 105 active+clean; 586 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.0 KiB/s wr, 93 op/s 2024-03-07T03:22:23.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:23 smithi148 bash[82222]: cluster 2024-03-07T03:22:22.316929+0000 mgr.y (mgr.34101) 227 : cluster 0 pgmap v425: 105 pgs: 105 active+clean; 586 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.0 KiB/s wr, 93 op/s 2024-03-07T03:22:23.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:23 smithi195 bash[66542]: cluster 2024-03-07T03:22:22.316929+0000 mgr.y (mgr.34101) 227 : cluster 0 pgmap v425: 105 pgs: 105 active+clean; 586 KiB data, 442 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.0 KiB/s wr, 93 op/s 2024-03-07T03:22:24.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:24 smithi148 bash[79819]: cluster 2024-03-07T03:22:23.226305+0000 mon.a (mon.0) 723 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2024-03-07T03:22:24.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:24 smithi148 bash[82222]: cluster 2024-03-07T03:22:23.226305+0000 mon.a (mon.0) 723 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2024-03-07T03:22:24.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:24 smithi195 bash[66542]: cluster 2024-03-07T03:22:23.226305+0000 mon.a (mon.0) 723 : cluster 0 osdmap e554: 8 total, 8 up, 8 in 2024-03-07T03:22:25.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:25 smithi148 bash[79819]: cluster 2024-03-07T03:22:24.235103+0000 mon.a (mon.0) 724 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2024-03-07T03:22:25.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:25 smithi148 bash[79819]: cluster 2024-03-07T03:22:24.317683+0000 mgr.y (mgr.34101) 228 : cluster 0 pgmap v428: 105 pgs: 105 active+clean; 4.6 MiB data, 450 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 1.0 MiB/s wr, 183 op/s 2024-03-07T03:22:25.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:25 smithi148 bash[82222]: cluster 2024-03-07T03:22:24.235103+0000 mon.a (mon.0) 724 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2024-03-07T03:22:25.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:25 smithi148 bash[82222]: cluster 2024-03-07T03:22:24.317683+0000 mgr.y (mgr.34101) 228 : cluster 0 pgmap v428: 105 pgs: 105 active+clean; 4.6 MiB data, 450 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 1.0 MiB/s wr, 183 op/s 2024-03-07T03:22:25.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:25 smithi195 bash[66542]: cluster 2024-03-07T03:22:24.235103+0000 mon.a (mon.0) 724 : cluster 0 osdmap e555: 8 total, 8 up, 8 in 2024-03-07T03:22:25.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:25 smithi195 bash[66542]: cluster 2024-03-07T03:22:24.317683+0000 mgr.y (mgr.34101) 228 : cluster 0 pgmap v428: 105 pgs: 105 active+clean; 4.6 MiB data, 450 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 1.0 MiB/s wr, 183 op/s 2024-03-07T03:22:26.270 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.WriteFullCopyup (4017 ms) 2024-03-07T03:22:26.270 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-03-07T03:22:26.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:26 smithi148 bash[79819]: cluster 2024-03-07T03:22:25.246269+0000 mon.a (mon.0) 725 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2024-03-07T03:22:26.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:26 smithi148 bash[82222]: cluster 2024-03-07T03:22:25.246269+0000 mon.a (mon.0) 725 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2024-03-07T03:22:26.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:26 smithi195 bash[66542]: cluster 2024-03-07T03:22:25.246269+0000 mon.a (mon.0) 725 : cluster 0 osdmap e556: 8 total, 8 up, 8 in 2024-03-07T03:22:27.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:22:27] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2024-03-07T03:22:27.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:27 smithi148 bash[79819]: cluster 2024-03-07T03:22:26.247106+0000 mon.a (mon.0) 726 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2024-03-07T03:22:27.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:27 smithi148 bash[79819]: cluster 2024-03-07T03:22:26.318407+0000 mgr.y (mgr.34101) 229 : cluster 0 pgmap v431: 105 pgs: 105 active+clean; 4.6 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 259 op/s 2024-03-07T03:22:27.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:27 smithi148 bash[82222]: cluster 2024-03-07T03:22:26.247106+0000 mon.a (mon.0) 726 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2024-03-07T03:22:27.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:27 smithi148 bash[82222]: cluster 2024-03-07T03:22:26.318407+0000 mgr.y (mgr.34101) 229 : cluster 0 pgmap v431: 105 pgs: 105 active+clean; 4.6 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 259 op/s 2024-03-07T03:22:27.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:27 smithi195 bash[66542]: cluster 2024-03-07T03:22:26.247106+0000 mon.a (mon.0) 726 : cluster 0 osdmap e557: 8 total, 8 up, 8 in 2024-03-07T03:22:27.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:27 smithi195 bash[66542]: cluster 2024-03-07T03:22:26.318407+0000 mgr.y (mgr.34101) 229 : cluster 0 pgmap v431: 105 pgs: 105 active+clean; 4.6 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.0 MiB/s wr, 259 op/s 2024-03-07T03:22:28.311 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2040 ms) 2024-03-07T03:22:28.311 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.TestCoR 2024-03-07T03:22:28.337 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_internal.cc:1313: Skipped 2024-03-07T03:22:28.338 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to disabled copy-on-read 2024-03-07T03:22:28.338 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestInternal.TestCoR (27 ms) 2024-03-07T03:22:28.338 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-03-07T03:22:28.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:28 smithi148 bash[79819]: cluster 2024-03-07T03:22:27.264870+0000 mon.a (mon.0) 727 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2024-03-07T03:22:28.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:28 smithi148 bash[79819]: audit 2024-03-07T03:22:27.312399+0000 mon.a (mon.0) 728 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:28.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:28 smithi148 bash[82222]: cluster 2024-03-07T03:22:27.264870+0000 mon.a (mon.0) 727 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2024-03-07T03:22:28.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:28 smithi148 bash[82222]: audit 2024-03-07T03:22:27.312399+0000 mon.a (mon.0) 728 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:28.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:28 smithi195 bash[66542]: cluster 2024-03-07T03:22:27.264870+0000 mon.a (mon.0) 727 : cluster 0 osdmap e558: 8 total, 8 up, 8 in 2024-03-07T03:22:28.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:28 smithi195 bash[66542]: audit 2024-03-07T03:22:27.312399+0000 mon.a (mon.0) 728 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:28.947 INFO:tasks.workunit.client.0.smithi148.stdout:made parent image "image398": 4194304K (1024 * 4K) 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout:generated random write map: 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [2 , 8946 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [27 , 112976 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [28 , 116111 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [43 , 178131 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [49 , 203343 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [59 , 243282 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [70 , 288077 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [91 , 373614 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [92 , 378567 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [96 , 395865 ] 2024-03-07T03:22:28.948 INFO:tasks.workunit.client.0.smithi148.stdout: [103 , 424386 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [104 , 426940 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [109 , 448446 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [123 , 506265 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [134 , 550290 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [137 , 561769 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [146 , 600023 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [167 , 686706 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [175 , 717229 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [184 , 754473 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [190 , 779375 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [191 , 783489 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [207 , 851218 ] 2024-03-07T03:22:28.949 INFO:tasks.workunit.client.0.smithi148.stdout: [213 , 874172 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [215 , 882840 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [219 , 897683 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [225 , 921714 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [229 , 939522 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [231 , 948651 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [253 , 1036381 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [264 , 1083013 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [266 , 1092391 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [271 , 1110247 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [275 , 1126580 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [277 , 1137478 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [288 , 1182892 ] 2024-03-07T03:22:28.950 INFO:tasks.workunit.client.0.smithi148.stdout: [295 , 1208856 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [299 , 1225935 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [307 , 1260565 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [317 , 1299993 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [348 , 1427753 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [354 , 1450528 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [360 , 1475278 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [362 , 1485546 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [363 , 1487659 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [370 , 1515977 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [384 , 1575656 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [390 , 1599339 ] 2024-03-07T03:22:28.951 INFO:tasks.workunit.client.0.smithi148.stdout: [391 , 1602676 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [407 , 1667099 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [415 , 1701803 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [416 , 1704326 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [433 , 1776631 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [436 , 1787436 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [444 , 1820624 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [445 , 1823224 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [458 , 1877560 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [467 , 1914665 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [485 , 1986601 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [516 , 2115659 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [536 , 2197445 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [556 , 2277520 ] 2024-03-07T03:22:28.952 INFO:tasks.workunit.client.0.smithi148.stdout: [571 , 2339865 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [578 , 2368168 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [583 , 2388200 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [612 , 2509726 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [618 , 2533476 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [632 , 2591214 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [633 , 2596267 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [653 , 2675646 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [672 , 2754250 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [722 , 2957640 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [732 , 3001715 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [740 , 3034472 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [749 , 3070765 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [785 , 3217661 ] 2024-03-07T03:22:28.953 INFO:tasks.workunit.client.0.smithi148.stdout: [808 , 3311783 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [836 , 3425950 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [837 , 3429856 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [841 , 3445071 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [842 , 3450335 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [855 , 3505634 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [856 , 3506185 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [885 , 3625788 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [892 , 3654692 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [904 , 3703610 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [907 , 3718126 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [918 , 3761994 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [921 , 3775103 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [965 , 3955161 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [970 , 3974915 ] 2024-03-07T03:22:28.954 INFO:tasks.workunit.client.0.smithi148.stdout: [978 , 4009115 ] 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: [984 , 4032934 ] 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout:write data based on random map 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-2 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-27 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-28 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-43 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-49 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-59 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-70 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-91 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-92 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-96 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-103 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-104 2024-03-07T03:22:28.955 INFO:tasks.workunit.client.0.smithi148.stdout: write object-109 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-123 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-134 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-137 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-146 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-167 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-175 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-184 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-190 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-191 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-207 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-213 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-215 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-219 2024-03-07T03:22:28.956 INFO:tasks.workunit.client.0.smithi148.stdout: write object-225 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-229 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-231 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-253 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-264 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-266 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-271 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-275 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-277 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-288 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-295 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-299 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-307 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-317 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-348 2024-03-07T03:22:28.957 INFO:tasks.workunit.client.0.smithi148.stdout: write object-354 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-360 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-362 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-363 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-370 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-384 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-390 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-391 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-407 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-415 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-416 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-433 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-436 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-444 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-445 2024-03-07T03:22:28.958 INFO:tasks.workunit.client.0.smithi148.stdout: write object-458 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-467 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-485 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-516 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-536 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-556 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-571 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-578 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-583 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-612 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-618 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-632 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-633 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-653 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-672 2024-03-07T03:22:28.959 INFO:tasks.workunit.client.0.smithi148.stdout: write object-722 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-732 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-740 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-749 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-785 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-808 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-836 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-837 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-841 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-842 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-855 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-856 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-885 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-892 2024-03-07T03:22:28.960 INFO:tasks.workunit.client.0.smithi148.stdout: write object-904 2024-03-07T03:22:28.961 INFO:tasks.workunit.client.0.smithi148.stdout: write object-907 2024-03-07T03:22:28.961 INFO:tasks.workunit.client.0.smithi148.stdout: write object-918 2024-03-07T03:22:28.961 INFO:tasks.workunit.client.0.smithi148.stdout: write object-921 2024-03-07T03:22:28.961 INFO:tasks.workunit.client.0.smithi148.stdout: write object-965 2024-03-07T03:22:28.961 INFO:tasks.workunit.client.0.smithi148.stdout: write object-970 2024-03-07T03:22:28.961 INFO:tasks.workunit.client.0.smithi148.stdout: write object-978 2024-03-07T03:22:29.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[79819]: cluster 2024-03-07T03:22:28.273997+0000 mon.a (mon.0) 729 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2024-03-07T03:22:29.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[79819]: cluster 2024-03-07T03:22:28.319252+0000 mgr.y (mgr.34101) 230 : cluster 0 pgmap v434: 105 pgs: 105 active+clean; 4.6 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.0 MiB/s wr, 233 op/s 2024-03-07T03:22:29.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[82222]: cluster 2024-03-07T03:22:28.273997+0000 mon.a (mon.0) 729 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2024-03-07T03:22:29.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[82222]: cluster 2024-03-07T03:22:28.319252+0000 mgr.y (mgr.34101) 230 : cluster 0 pgmap v434: 105 pgs: 105 active+clean; 4.6 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.0 MiB/s wr, 233 op/s 2024-03-07T03:22:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:29 smithi195 bash[66542]: cluster 2024-03-07T03:22:28.273997+0000 mon.a (mon.0) 729 : cluster 0 osdmap e559: 8 total, 8 up, 8 in 2024-03-07T03:22:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:29 smithi195 bash[66542]: cluster 2024-03-07T03:22:28.319252+0000 mgr.y (mgr.34101) 230 : cluster 0 pgmap v434: 105 pgs: 105 active+clean; 4.6 MiB data, 463 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.0 MiB/s wr, 233 op/s 2024-03-07T03:22:29.647 INFO:tasks.workunit.client.0.smithi148.stdout: write object-984 2024-03-07T03:22:29.647 INFO:tasks.workunit.client.0.smithi148.stdout:verify written data by reading 2024-03-07T03:22:29.647 INFO:tasks.workunit.client.0.smithi148.stdout: read object-2 2024-03-07T03:22:29.648 INFO:tasks.workunit.client.0.smithi148.stdout:made snapshot "image398@parent_snap" and protect it 2024-03-07T03:22:29.648 INFO:tasks.workunit.client.0.smithi148.stdout:made and opened clone "image399" 2024-03-07T03:22:29.648 INFO:tasks.workunit.client.0.smithi148.stdout:flattening clone: "image399" 2024-03-07T03:22:29.648 INFO:tasks.workunit.client.0.smithi148.stdout:check whether child image has the same set of objects as parent 2024-03-07T03:22:29.648 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1309 ms) 2024-03-07T03:22:29.648 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-03-07T03:22:29.751 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.PoolMetadataConfApply (105 ms) 2024-03-07T03:22:29.751 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.Sparsify 2024-03-07T03:22:29.783 INFO:tasks.workunit.client.0.smithi148.stdout:sparsify_supported=1 2024-03-07T03:22:29.784 INFO:tasks.workunit.client.0.smithi148.stdout:sparse_read_supported=1 2024-03-07T03:22:29.821 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.Sparsify (69 ms) 2024-03-07T03:22:29.821 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.SparsifyClone 2024-03-07T03:22:29.844 INFO:tasks.workunit.client.0.smithi148.stdout:sparsify_supported=1 2024-03-07T03:22:30.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[72584]: [07/Mar/2024:03:22:29] ENGINE Bus STOPPING 2024-03-07T03:22:30.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[72584]: [07/Mar/2024:03:22:29] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:22:30.040 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[72584]: [07/Mar/2024:03:22:29] ENGINE Bus STOPPED 2024-03-07T03:22:30.040 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[72584]: [07/Mar/2024:03:22:29] ENGINE Bus STARTING 2024-03-07T03:22:30.040 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[72584]: [07/Mar/2024:03:22:29] ENGINE Serving on http://:::9283 2024-03-07T03:22:30.040 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:29 smithi148 bash[72584]: [07/Mar/2024:03:22:29] ENGINE Bus STARTED 2024-03-07T03:22:30.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[79819]: cluster 2024-03-07T03:22:29.283709+0000 mon.a (mon.0) 730 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2024-03-07T03:22:30.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[79819]: audit 2024-03-07T03:22:29.690178+0000 mon.a (mon.0) 731 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[79819]: audit 2024-03-07T03:22:29.717949+0000 mon.a (mon.0) 732 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[79819]: audit 2024-03-07T03:22:29.719759+0000 mon.a (mon.0) 733 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[79819]: audit 2024-03-07T03:22:29.720577+0000 mon.a (mon.0) 734 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[79819]: audit 2024-03-07T03:22:29.726324+0000 mon.a (mon.0) 735 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[82222]: cluster 2024-03-07T03:22:29.283709+0000 mon.a (mon.0) 730 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[82222]: audit 2024-03-07T03:22:29.690178+0000 mon.a (mon.0) 731 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[82222]: audit 2024-03-07T03:22:29.717949+0000 mon.a (mon.0) 732 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:30.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[82222]: audit 2024-03-07T03:22:29.719759+0000 mon.a (mon.0) 733 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:30.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[82222]: audit 2024-03-07T03:22:29.720577+0000 mon.a (mon.0) 734 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:30.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:30 smithi148 bash[82222]: audit 2024-03-07T03:22:29.726324+0000 mon.a (mon.0) 735 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:30.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:30 smithi195 bash[66542]: cluster 2024-03-07T03:22:29.283709+0000 mon.a (mon.0) 730 : cluster 0 osdmap e560: 8 total, 8 up, 8 in 2024-03-07T03:22:30.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:30 smithi195 bash[66542]: audit 2024-03-07T03:22:29.690178+0000 mon.a (mon.0) 731 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:22:30.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:30 smithi195 bash[66542]: audit 2024-03-07T03:22:29.717949+0000 mon.a (mon.0) 732 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:30.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:30 smithi195 bash[66542]: audit 2024-03-07T03:22:29.719759+0000 mon.a (mon.0) 733 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:30.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:30 smithi195 bash[66542]: audit 2024-03-07T03:22:29.720577+0000 mon.a (mon.0) 734 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:30.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:30 smithi195 bash[66542]: audit 2024-03-07T03:22:29.726324+0000 mon.a (mon.0) 735 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:31.120 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade resume' 2024-03-07T03:22:31.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:31 smithi148 bash[79819]: cluster 2024-03-07T03:22:30.320093+0000 mgr.y (mgr.34101) 231 : cluster 0 pgmap v436: 105 pgs: 105 active+clean; 4.7 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 70 KiB/s wr, 436 op/s 2024-03-07T03:22:31.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:31 smithi148 bash[79819]: cluster 2024-03-07T03:22:30.733955+0000 mon.a (mon.0) 736 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2024-03-07T03:22:31.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:31 smithi148 bash[82222]: cluster 2024-03-07T03:22:30.320093+0000 mgr.y (mgr.34101) 231 : cluster 0 pgmap v436: 105 pgs: 105 active+clean; 4.7 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 70 KiB/s wr, 436 op/s 2024-03-07T03:22:31.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:31 smithi148 bash[82222]: cluster 2024-03-07T03:22:30.733955+0000 mon.a (mon.0) 736 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2024-03-07T03:22:31.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:31 smithi195 bash[66542]: cluster 2024-03-07T03:22:30.320093+0000 mgr.y (mgr.34101) 231 : cluster 0 pgmap v436: 105 pgs: 105 active+clean; 4.7 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 70 KiB/s wr, 436 op/s 2024-03-07T03:22:31.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:31 smithi195 bash[66542]: cluster 2024-03-07T03:22:30.733955+0000 mon.a (mon.0) 736 : cluster 0 osdmap e561: 8 total, 8 up, 8 in 2024-03-07T03:22:31.755 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.SparsifyClone (1935 ms) 2024-03-07T03:22:31.755 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestInternal.MissingDataPool 2024-03-07T03:22:32.365 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestInternal.MissingDataPool (610 ms) 2024-03-07T03:22:32.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:22:32 smithi195 bash[59134]: ts=2024-03-07T03:22:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:33.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:32 smithi148 bash[79819]: cluster 2024-03-07T03:22:31.735241+0000 mon.a (mon.0) 737 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2024-03-07T03:22:33.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:32 smithi148 bash[79819]: cluster 2024-03-07T03:22:32.108972+0000 mon.a (mon.0) 738 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2024-03-07T03:22:33.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:32 smithi148 bash[79819]: cluster 2024-03-07T03:22:32.320665+0000 mgr.y (mgr.34101) 232 : cluster 0 pgmap v440: 105 pgs: 105 active+clean; 4.7 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 330 KiB/s rd, 120 KiB/s wr, 530 op/s 2024-03-07T03:22:33.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:32 smithi148 bash[82222]: cluster 2024-03-07T03:22:31.735241+0000 mon.a (mon.0) 737 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2024-03-07T03:22:33.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:32 smithi148 bash[82222]: cluster 2024-03-07T03:22:32.108972+0000 mon.a (mon.0) 738 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2024-03-07T03:22:33.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:32 smithi148 bash[82222]: cluster 2024-03-07T03:22:32.320665+0000 mgr.y (mgr.34101) 232 : cluster 0 pgmap v440: 105 pgs: 105 active+clean; 4.7 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 330 KiB/s rd, 120 KiB/s wr, 530 op/s 2024-03-07T03:22:33.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:32 smithi195 bash[66542]: cluster 2024-03-07T03:22:31.735241+0000 mon.a (mon.0) 737 : cluster 0 osdmap e562: 8 total, 8 up, 8 in 2024-03-07T03:22:33.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:32 smithi195 bash[66542]: cluster 2024-03-07T03:22:32.108972+0000 mon.a (mon.0) 738 : cluster 0 osdmap e563: 8 total, 8 up, 8 in 2024-03-07T03:22:33.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:32 smithi195 bash[66542]: cluster 2024-03-07T03:22:32.320665+0000 mgr.y (mgr.34101) 232 : cluster 0 pgmap v440: 105 pgs: 105 active+clean; 4.7 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 330 KiB/s rd, 120 KiB/s wr, 530 op/s 2024-03-07T03:22:33.112 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 34 tests from TestInternal (35109 ms total) 2024-03-07T03:22:33.112 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:22:33.113 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 17 tests from TestImageWatcher 2024-03-07T03:22:34.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:34 smithi148 bash[79819]: cluster 2024-03-07T03:22:33.119856+0000 mon.a (mon.0) 739 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2024-03-07T03:22:34.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:34 smithi148 bash[82222]: cluster 2024-03-07T03:22:33.119856+0000 mon.a (mon.0) 739 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2024-03-07T03:22:34.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:34 smithi195 bash[66542]: cluster 2024-03-07T03:22:33.119856+0000 mon.a (mon.0) 739 : cluster 0 osdmap e564: 8 total, 8 up, 8 in 2024-03-07T03:22:35.118 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-03-07T03:22:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[79819]: cluster 2024-03-07T03:22:34.125054+0000 mon.a (mon.0) 740 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2024-03-07T03:22:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[79819]: audit 2024-03-07T03:22:34.128746+0000 mon.c (mon.2) 22 : audit 1 from='client.? 172.21.15.148:0/2991141258' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[79819]: audit 2024-03-07T03:22:34.143472+0000 mon.a (mon.0) 741 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[79819]: cluster 2024-03-07T03:22:34.321349+0000 mgr.y (mgr.34101) 233 : cluster 0 pgmap v443: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 450 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:35.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[79819]: audit 2024-03-07T03:22:35.120058+0000 mon.a (mon.0) 742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[82222]: cluster 2024-03-07T03:22:34.125054+0000 mon.a (mon.0) 740 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2024-03-07T03:22:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[82222]: audit 2024-03-07T03:22:34.128746+0000 mon.c (mon.2) 22 : audit 1 from='client.? 172.21.15.148:0/2991141258' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[82222]: audit 2024-03-07T03:22:34.143472+0000 mon.a (mon.0) 741 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[82222]: cluster 2024-03-07T03:22:34.321349+0000 mgr.y (mgr.34101) 233 : cluster 0 pgmap v443: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 450 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:35.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:35 smithi148 bash[82222]: audit 2024-03-07T03:22:35.120058+0000 mon.a (mon.0) 742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:35.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:35 smithi195 bash[66542]: cluster 2024-03-07T03:22:34.125054+0000 mon.a (mon.0) 740 : cluster 0 osdmap e565: 8 total, 8 up, 8 in 2024-03-07T03:22:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:35 smithi195 bash[66542]: audit 2024-03-07T03:22:34.128746+0000 mon.c (mon.2) 22 : audit 1 from='client.? 172.21.15.148:0/2991141258' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:35 smithi195 bash[66542]: audit 2024-03-07T03:22:34.143472+0000 mon.a (mon.0) 741 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:35 smithi195 bash[66542]: cluster 2024-03-07T03:22:34.321349+0000 mgr.y (mgr.34101) 233 : cluster 0 pgmap v443: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 450 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:35 smithi195 bash[66542]: audit 2024-03-07T03:22:35.120058+0000 mon.a (mon.0) 742 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:35.860 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:22:36.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:36 smithi148 bash[79819]: cluster 2024-03-07T03:22:35.132126+0000 mon.a (mon.0) 743 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2024-03-07T03:22:36.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:36 smithi148 bash[79819]: cluster 2024-03-07T03:22:35.133281+0000 mon.a (mon.0) 744 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:36.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:36 smithi148 bash[79819]: cluster 2024-03-07T03:22:36.129605+0000 mon.a (mon.0) 745 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2024-03-07T03:22:36.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:36 smithi148 bash[82222]: cluster 2024-03-07T03:22:35.132126+0000 mon.a (mon.0) 743 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2024-03-07T03:22:36.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:36 smithi148 bash[82222]: cluster 2024-03-07T03:22:35.133281+0000 mon.a (mon.0) 744 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:36.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:36 smithi148 bash[82222]: cluster 2024-03-07T03:22:36.129605+0000 mon.a (mon.0) 745 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2024-03-07T03:22:36.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:36 smithi195 bash[66542]: cluster 2024-03-07T03:22:35.132126+0000 mon.a (mon.0) 743 : cluster 0 osdmap e566: 8 total, 8 up, 8 in 2024-03-07T03:22:36.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:36 smithi195 bash[66542]: cluster 2024-03-07T03:22:35.133281+0000 mon.a (mon.0) 744 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:36.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:36 smithi195 bash[66542]: cluster 2024-03-07T03:22:36.129605+0000 mon.a (mon.0) 745 : cluster 0 osdmap e567: 8 total, 8 up, 8 in 2024-03-07T03:22:37.158 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2039 ms) 2024-03-07T03:22:37.158 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-03-07T03:22:37.191 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyFlatten (33 ms) 2024-03-07T03:22:37.191 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-03-07T03:22:37.221 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyResize (30 ms) 2024-03-07T03:22:37.221 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-03-07T03:22:37.242 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:22:37] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2024-03-07T03:22:37.242 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:37 smithi148 bash[79819]: cluster 2024-03-07T03:22:36.322226+0000 mgr.y (mgr.34101) 234 : cluster 0 pgmap v446: 105 pgs: 21 unknown, 84 active+clean; 582 KiB data, 445 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:37.242 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:37 smithi148 bash[79819]: cluster 2024-03-07T03:22:37.129178+0000 mon.a (mon.0) 746 : cluster 0 osdmap e568: 8 total, 8 up, 8 in 2024-03-07T03:22:37.242 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:37 smithi148 bash[82222]: cluster 2024-03-07T03:22:36.322226+0000 mgr.y (mgr.34101) 234 : cluster 0 pgmap v446: 105 pgs: 21 unknown, 84 active+clean; 582 KiB data, 445 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:37.242 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:37 smithi148 bash[82222]: cluster 2024-03-07T03:22:37.129178+0000 mon.a (mon.0) 746 : cluster 0 osdmap e568: 8 total, 8 up, 8 in 2024-03-07T03:22:37.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (37 ms) 2024-03-07T03:22:37.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-03-07T03:22:37.294 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (36 ms) 2024-03-07T03:22:37.294 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-03-07T03:22:37.329 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (35 ms) 2024-03-07T03:22:37.329 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-03-07T03:22:37.364 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (35 ms) 2024-03-07T03:22:37.364 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-03-07T03:22:37.402 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifySnapRename (38 ms) 2024-03-07T03:22:37.402 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-03-07T03:22:37.438 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (36 ms) 2024-03-07T03:22:37.438 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-03-07T03:22:37.480 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (42 ms) 2024-03-07T03:22:37.480 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-03-07T03:22:37.518 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (38 ms) 2024-03-07T03:22:37.518 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-03-07T03:22:37.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:37 smithi195 bash[66542]: cluster 2024-03-07T03:22:36.322226+0000 mgr.y (mgr.34101) 234 : cluster 0 pgmap v446: 105 pgs: 21 unknown, 84 active+clean; 582 KiB data, 445 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:37.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:37 smithi195 bash[66542]: cluster 2024-03-07T03:22:37.129178+0000 mon.a (mon.0) 746 : cluster 0 osdmap e568: 8 total, 8 up, 8 in 2024-03-07T03:22:37.571 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (53 ms) 2024-03-07T03:22:37.571 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-03-07T03:22:37.618 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyRename (47 ms) 2024-03-07T03:22:37.618 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-03-07T03:22:37.650 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (32 ms) 2024-03-07T03:22:37.651 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-03-07T03:22:37.692 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (42 ms) 2024-03-07T03:22:37.692 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-03-07T03:22:37.735 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (43 ms) 2024-03-07T03:22:37.735 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-03-07T03:22:37.774 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (39 ms) 2024-03-07T03:22:38.173 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 17 tests from TestImageWatcher (2656 ms total) 2024-03-07T03:22:38.173 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:22:38.173 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 168 tests from TestLibRBD 2024-03-07T03:22:38.227 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-03-07T03:22:38.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:38 smithi148 bash[79819]: cluster 2024-03-07T03:22:37.159385+0000 mon.a (mon.0) 747 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:38.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:38 smithi148 bash[82222]: cluster 2024-03-07T03:22:37.159385+0000 mon.a (mon.0) 747 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:38.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:38 smithi195 bash[66542]: cluster 2024-03-07T03:22:37.159385+0000 mon.a (mon.0) 747 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:38.997 INFO:teuthology.orchestra.run.smithi148.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:39.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[79819]: cluster 2024-03-07T03:22:38.183254+0000 mon.a (mon.0) 748 : cluster 0 osdmap e569: 8 total, 8 up, 8 in 2024-03-07T03:22:39.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[79819]: cluster 2024-03-07T03:22:38.323039+0000 mgr.y (mgr.34101) 235 : cluster 0 pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:39.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[79819]: audit 2024-03-07T03:22:39.011137+0000 mon.a (mon.0) 749 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:39.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[79819]: audit 2024-03-07T03:22:39.035542+0000 mon.a (mon.0) 750 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:39.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[82222]: cluster 2024-03-07T03:22:38.183254+0000 mon.a (mon.0) 748 : cluster 0 osdmap e569: 8 total, 8 up, 8 in 2024-03-07T03:22:39.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[82222]: cluster 2024-03-07T03:22:38.323039+0000 mgr.y (mgr.34101) 235 : cluster 0 pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:39.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[82222]: audit 2024-03-07T03:22:39.011137+0000 mon.a (mon.0) 749 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:39.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:39 smithi148 bash[82222]: audit 2024-03-07T03:22:39.035542+0000 mon.a (mon.0) 750 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:39 smithi195 bash[66542]: cluster 2024-03-07T03:22:38.183254+0000 mon.a (mon.0) 748 : cluster 0 osdmap e569: 8 total, 8 up, 8 in 2024-03-07T03:22:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:39 smithi195 bash[66542]: cluster 2024-03-07T03:22:38.323039+0000 mgr.y (mgr.34101) 235 : cluster 0 pgmap v449: 73 pgs: 73 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:39 smithi195 bash[66542]: audit 2024-03-07T03:22:39.011137+0000 mon.a (mon.0) 749 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:39 smithi195 bash[66542]: audit 2024-03-07T03:22:39.035542+0000 mon.a (mon.0) 750 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:39.732 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'echo wait for half of osds to upgrade' 2024-03-07T03:22:40.217 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[79819]: audit 2024-03-07T03:22:39.003031+0000 mgr.y (mgr.34101) 236 : audit 0 from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[79819]: cephadm 2024-03-07T03:22:39.003705+0000 mgr.y (mgr.34101) 237 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[79819]: cluster 2024-03-07T03:22:39.212415+0000 mon.a (mon.0) 751 : cluster 0 osdmap e570: 8 total, 8 up, 8 in 2024-03-07T03:22:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[79819]: audit 2024-03-07T03:22:39.214855+0000 mon.a (mon.0) 752 : audit 1 from='client.? 172.21.15.148:0/4090902618' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[82222]: audit 2024-03-07T03:22:39.003031+0000 mgr.y (mgr.34101) 236 : audit 0 from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[82222]: cephadm 2024-03-07T03:22:39.003705+0000 mgr.y (mgr.34101) 237 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[82222]: cluster 2024-03-07T03:22:39.212415+0000 mon.a (mon.0) 751 : cluster 0 osdmap e570: 8 total, 8 up, 8 in 2024-03-07T03:22:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:40 smithi148 bash[82222]: audit 2024-03-07T03:22:39.214855+0000 mon.a (mon.0) 752 : audit 1 from='client.? 172.21.15.148:0/4090902618' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:40.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:40 smithi195 bash[66542]: audit 2024-03-07T03:22:39.003031+0000 mgr.y (mgr.34101) 236 : audit 0 from='client.34314 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:22:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:40 smithi195 bash[66542]: cephadm 2024-03-07T03:22:39.003705+0000 mgr.y (mgr.34101) 237 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:22:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:40 smithi195 bash[66542]: cluster 2024-03-07T03:22:39.212415+0000 mon.a (mon.0) 751 : cluster 0 osdmap e570: 8 total, 8 up, 8 in 2024-03-07T03:22:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:40 smithi195 bash[66542]: audit 2024-03-07T03:22:39.214855+0000 mon.a (mon.0) 752 : audit 1 from='client.? 172.21.15.148:0/4090902618' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-15","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:41.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:41 smithi148 bash[79819]: audit 2024-03-07T03:22:40.207578+0000 mon.a (mon.0) 753 : audit 1 from='client.? 172.21.15.148:0/4090902618' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:41.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:41 smithi148 bash[79819]: cluster 2024-03-07T03:22:40.216499+0000 mon.a (mon.0) 754 : cluster 0 osdmap e571: 8 total, 8 up, 8 in 2024-03-07T03:22:41.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:41 smithi148 bash[79819]: cluster 2024-03-07T03:22:40.323700+0000 mgr.y (mgr.34101) 238 : cluster 0 pgmap v452: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:41 smithi148 bash[82222]: audit 2024-03-07T03:22:40.207578+0000 mon.a (mon.0) 753 : audit 1 from='client.? 172.21.15.148:0/4090902618' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:41 smithi148 bash[82222]: cluster 2024-03-07T03:22:40.216499+0000 mon.a (mon.0) 754 : cluster 0 osdmap e571: 8 total, 8 up, 8 in 2024-03-07T03:22:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:41 smithi148 bash[82222]: cluster 2024-03-07T03:22:40.323700+0000 mgr.y (mgr.34101) 238 : cluster 0 pgmap v452: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:41.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:41 smithi195 bash[66542]: audit 2024-03-07T03:22:40.207578+0000 mon.a (mon.0) 753 : audit 1 from='client.? 172.21.15.148:0/4090902618' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:41.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:41 smithi195 bash[66542]: cluster 2024-03-07T03:22:40.216499+0000 mon.a (mon.0) 754 : cluster 0 osdmap e571: 8 total, 8 up, 8 in 2024-03-07T03:22:41.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:41 smithi195 bash[66542]: cluster 2024-03-07T03:22:40.323700+0000 mgr.y (mgr.34101) 238 : cluster 0 pgmap v452: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:42.287 INFO:tasks.workunit.client.0.smithi148.stdout:image has size 2097152 and order 22 2024-03-07T03:22:42.287 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CreateAndStat (4060 ms) 2024-03-07T03:22:42.287 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-03-07T03:22:42.287 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.320 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (31 ms) 2024-03-07T03:22:42.320 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-03-07T03:22:42.320 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.356 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CreateAndStatPP (39 ms) 2024-03-07T03:22:42.356 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.GetId 2024-03-07T03:22:42.356 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.391 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.GetId (34 ms) 2024-03-07T03:22:42.391 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-03-07T03:22:42.391 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.440 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.GetIdPP (50 ms) 2024-03-07T03:22:42.440 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-03-07T03:22:42.440 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.467 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (27 ms) 2024-03-07T03:22:42.468 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-03-07T03:22:42.468 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.498 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (30 ms) 2024-03-07T03:22:42.498 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-03-07T03:22:42.498 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.528 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (30 ms) 2024-03-07T03:22:42.528 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-03-07T03:22:42.528 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:42 smithi148 bash[79819]: cluster 2024-03-07T03:22:41.243650+0000 mon.a (mon.0) 755 : cluster 0 osdmap e572: 8 total, 8 up, 8 in 2024-03-07T03:22:42.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:42 smithi148 bash[82222]: cluster 2024-03-07T03:22:41.243650+0000 mon.a (mon.0) 755 : cluster 0 osdmap e572: 8 total, 8 up, 8 in 2024-03-07T03:22:42.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:42 smithi195 bash[66542]: cluster 2024-03-07T03:22:41.243650+0000 mon.a (mon.0) 755 : cluster 0 osdmap e572: 8 total, 8 up, 8 in 2024-03-07T03:22:42.562 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (34 ms) 2024-03-07T03:22:42.562 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.OpenAio 2024-03-07T03:22:42.562 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:22:42 smithi195 bash[59134]: ts=2024-03-07T03:22:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:42.591 INFO:tasks.workunit.client.0.smithi148.stdout:image has size 2097152 and order 22 2024-03-07T03:22:42.591 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.OpenAio (30 ms) 2024-03-07T03:22:42.591 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-03-07T03:22:42.594 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.OpenAioFail (3 ms) 2024-03-07T03:22:42.594 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-03-07T03:22:42.595 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.636 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.OpenAioPP (41 ms) 2024-03-07T03:22:42.636 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-03-07T03:22:42.640 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.OpenAioFailPP (5 ms) 2024-03-07T03:22:42.640 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-03-07T03:22:42.641 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.722 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ResizeAndStat (81 ms) 2024-03-07T03:22:42.722 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-03-07T03:22:42.722 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.791 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (70 ms) 2024-03-07T03:22:42.791 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-03-07T03:22:42.791 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.871 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (80 ms) 2024-03-07T03:22:42.871 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-03-07T03:22:42.872 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:42.952 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (80 ms) 2024-03-07T03:22:42.952 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-03-07T03:22:43.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:43 smithi148 bash[79819]: cluster 2024-03-07T03:22:42.243116+0000 mon.a (mon.0) 756 : cluster 0 osdmap e573: 8 total, 8 up, 8 in 2024-03-07T03:22:43.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:43 smithi148 bash[79819]: audit 2024-03-07T03:22:42.311942+0000 mon.a (mon.0) 757 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:43.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:43 smithi148 bash[79819]: cluster 2024-03-07T03:22:42.324128+0000 mgr.y (mgr.34101) 239 : cluster 0 pgmap v455: 105 pgs: 26 unknown, 79 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:43.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:43 smithi148 bash[82222]: cluster 2024-03-07T03:22:42.243116+0000 mon.a (mon.0) 756 : cluster 0 osdmap e573: 8 total, 8 up, 8 in 2024-03-07T03:22:43.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:43 smithi148 bash[82222]: audit 2024-03-07T03:22:42.311942+0000 mon.a (mon.0) 757 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:43.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:43 smithi148 bash[82222]: cluster 2024-03-07T03:22:42.324128+0000 mgr.y (mgr.34101) 239 : cluster 0 pgmap v455: 105 pgs: 26 unknown, 79 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:43.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:43 smithi195 bash[66542]: cluster 2024-03-07T03:22:42.243116+0000 mon.a (mon.0) 756 : cluster 0 osdmap e573: 8 total, 8 up, 8 in 2024-03-07T03:22:43.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:43 smithi195 bash[66542]: audit 2024-03-07T03:22:42.311942+0000 mon.a (mon.0) 757 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:43.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:43 smithi195 bash[66542]: cluster 2024-03-07T03:22:42.324128+0000 mgr.y (mgr.34101) 239 : cluster 0 pgmap v455: 105 pgs: 26 unknown, 79 active+clean; 582 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:22:44.000 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:22:44.514 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:44 smithi148 bash[79819]: cluster 2024-03-07T03:22:43.489441+0000 mon.a (mon.0) 758 : cluster 0 osdmap e574: 8 total, 8 up, 8 in 2024-03-07T03:22:44.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:44 smithi148 bash[79819]: audit 2024-03-07T03:22:43.494665+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.148:0/1615727494' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:44 smithi148 bash[82222]: cluster 2024-03-07T03:22:43.489441+0000 mon.a (mon.0) 758 : cluster 0 osdmap e574: 8 total, 8 up, 8 in 2024-03-07T03:22:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:44 smithi148 bash[82222]: audit 2024-03-07T03:22:43.494665+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.148:0/1615727494' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:44.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:44 smithi195 bash[66542]: cluster 2024-03-07T03:22:43.489441+0000 mon.a (mon.0) 758 : cluster 0 osdmap e574: 8 total, 8 up, 8 in 2024-03-07T03:22:44.811 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:44 smithi195 bash[66542]: audit 2024-03-07T03:22:43.494665+0000 mon.a (mon.0) 759 : audit 1 from='client.? 172.21.15.148:0/1615727494' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-16","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:45.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[72584]: debug 2024-03-07T03:22:45.387+0000 7f6cc280d700 -1 mgr.server reply reply (11) Resource temporarily unavailable 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:22:45.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[72584]: debug 2024-03-07T03:22:45.387+0000 7f6cc280d700 -1 mgr.server reply reply (16) Device or resource busy 32 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: cluster 2024-03-07T03:22:44.324607+0000 mgr.y (mgr.34101) 240 : cluster 0 pgmap v457: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 15 KiB/s wr, 146 op/s 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: cluster 2024-03-07T03:22:44.483089+0000 mon.a (mon.0) 760 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:44.491420+0000 mon.a (mon.0) 761 : audit 1 from='client.? 172.21.15.148:0/1615727494' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: cluster 2024-03-07T03:22:44.499478+0000 mon.a (mon.0) 762 : cluster 0 osdmap e575: 8 total, 8 up, 8 in 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:44.771219+0000 mon.a (mon.0) 763 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:44.777032+0000 mon.a (mon.0) 764 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.296808+0000 mon.a (mon.0) 765 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.297997+0000 mon.a (mon.0) 766 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.305419+0000 mon.a (mon.0) 767 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.378317+0000 mon.a (mon.0) 768 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.382832+0000 mon.a (mon.0) 769 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.385862+0000 mon.a (mon.0) 770 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.387892+0000 mon.a (mon.0) 771 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[79819]: audit 2024-03-07T03:22:45.390841+0000 mon.a (mon.0) 772 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: cluster 2024-03-07T03:22:44.324607+0000 mgr.y (mgr.34101) 240 : cluster 0 pgmap v457: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 15 KiB/s wr, 146 op/s 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: cluster 2024-03-07T03:22:44.483089+0000 mon.a (mon.0) 760 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:44.491420+0000 mon.a (mon.0) 761 : audit 1 from='client.? 172.21.15.148:0/1615727494' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: cluster 2024-03-07T03:22:44.499478+0000 mon.a (mon.0) 762 : cluster 0 osdmap e575: 8 total, 8 up, 8 in 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:44.771219+0000 mon.a (mon.0) 763 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:44.777032+0000 mon.a (mon.0) 764 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.296808+0000 mon.a (mon.0) 765 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.297997+0000 mon.a (mon.0) 766 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:45.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.305419+0000 mon.a (mon.0) 767 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.378317+0000 mon.a (mon.0) 768 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:45.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.382832+0000 mon.a (mon.0) 769 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.385862+0000 mon.a (mon.0) 770 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.387892+0000 mon.a (mon.0) 771 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:45 smithi148 bash[82222]: audit 2024-03-07T03:22:45.390841+0000 mon.a (mon.0) 772 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:22:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: cluster 2024-03-07T03:22:44.324607+0000 mgr.y (mgr.34101) 240 : cluster 0 pgmap v457: 137 pgs: 32 unknown, 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 102 KiB/s rd, 15 KiB/s wr, 146 op/s 2024-03-07T03:22:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: cluster 2024-03-07T03:22:44.483089+0000 mon.a (mon.0) 760 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:44.491420+0000 mon.a (mon.0) 761 : audit 1 from='client.? 172.21.15.148:0/1615727494' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: cluster 2024-03-07T03:22:44.499478+0000 mon.a (mon.0) 762 : cluster 0 osdmap e575: 8 total, 8 up, 8 in 2024-03-07T03:22:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:44.771219+0000 mon.a (mon.0) 763 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:44.777032+0000 mon.a (mon.0) 764 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.296808+0000 mon.a (mon.0) 765 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.297997+0000 mon.a (mon.0) 766 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.305419+0000 mon.a (mon.0) 767 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.378317+0000 mon.a (mon.0) 768 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.382832+0000 mon.a (mon.0) 769 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.385862+0000 mon.a (mon.0) 770 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.387892+0000 mon.a (mon.0) 771 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:22:45.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:45 smithi195 bash[66542]: audit 2024-03-07T03:22:45.390841+0000 mon.a (mon.0) 772 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:22:46.542 INFO:tasks.workunit.client.0.smithi148.stdout:image: image18 2024-03-07T03:22:46.542 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image18 2024-03-07T03:22:46.542 INFO:tasks.workunit.client.0.smithi148.stdout:found image18 2024-03-07T03:22:46.542 INFO:tasks.workunit.client.0.smithi148.stdout:erased image18 2024-03-07T03:22:46.542 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:46.722 INFO:teuthology.orchestra.run.smithi148.stdout:wait for half of osds to upgrade 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:image: image18 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:image: image19 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image18 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:found image18 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:erased image18 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image19 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:found image19 2024-03-07T03:22:46.722 INFO:tasks.workunit.client.0.smithi148.stdout:erased image19 2024-03-07T03:22:46.723 INFO:tasks.workunit.client.0.smithi148.stdout:image: image19 2024-03-07T03:22:46.723 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image19 2024-03-07T03:22:46.723 INFO:tasks.workunit.client.0.smithi148.stdout:found image19 2024-03-07T03:22:46.723 INFO:tasks.workunit.client.0.smithi148.stdout:erased image19 2024-03-07T03:22:46.723 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3770 ms) 2024-03-07T03:22:46.723 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-03-07T03:22:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:46 smithi148 bash[82222]: audit 2024-03-07T03:22:45.391082+0000 mgr.y (mgr.34101) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:22:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:46 smithi148 bash[82222]: cephadm 2024-03-07T03:22:45.392648+0000 mgr.y (mgr.34101) 242 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:22:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:46 smithi148 bash[82222]: cluster 2024-03-07T03:22:45.499890+0000 mon.a (mon.0) 773 : cluster 0 osdmap e576: 8 total, 8 up, 8 in 2024-03-07T03:22:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:46 smithi148 bash[79819]: audit 2024-03-07T03:22:45.391082+0000 mgr.y (mgr.34101) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:22:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:46 smithi148 bash[79819]: cephadm 2024-03-07T03:22:45.392648+0000 mgr.y (mgr.34101) 242 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:22:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:46 smithi148 bash[79819]: cluster 2024-03-07T03:22:45.499890+0000 mon.a (mon.0) 773 : cluster 0 osdmap e576: 8 total, 8 up, 8 in 2024-03-07T03:22:46.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:46 smithi195 bash[66542]: audit 2024-03-07T03:22:45.391082+0000 mgr.y (mgr.34101) 241 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:22:46.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:46 smithi195 bash[66542]: cephadm 2024-03-07T03:22:45.392648+0000 mgr.y (mgr.34101) 242 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:22:46.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:46 smithi195 bash[66542]: cluster 2024-03-07T03:22:45.499890+0000 mon.a (mon.0) 773 : cluster 0 osdmap e576: 8 total, 8 up, 8 in 2024-03-07T03:22:47.504 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:22:47] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.43.0" 2024-03-07T03:22:47.522 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-03-07T03:22:47.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[82222]: cluster 2024-03-07T03:22:46.325406+0000 mgr.y (mgr.34101) 243 : cluster 0 pgmap v460: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 20 KiB/s wr, 245 op/s 2024-03-07T03:22:47.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[82222]: cluster 2024-03-07T03:22:46.502244+0000 mon.a (mon.0) 774 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:47.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[82222]: cluster 2024-03-07T03:22:46.524519+0000 mon.a (mon.0) 775 : cluster 0 osdmap e577: 8 total, 8 up, 8 in 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[82222]: cluster 2024-03-07T03:22:47.198843+0000 mon.a (mon.0) 776 : cluster 0 osdmap e578: 8 total, 8 up, 8 in 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[82222]: audit 2024-03-07T03:22:47.201125+0000 mon.c (mon.2) 23 : audit 1 from='client.? 172.21.15.148:0/482266029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[82222]: audit 2024-03-07T03:22:47.211624+0000 mon.a (mon.0) 777 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[82222]: audit 2024-03-07T03:22:47.477822+0000 mon.a (mon.0) 778 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[79819]: cluster 2024-03-07T03:22:46.325406+0000 mgr.y (mgr.34101) 243 : cluster 0 pgmap v460: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 20 KiB/s wr, 245 op/s 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[79819]: cluster 2024-03-07T03:22:46.502244+0000 mon.a (mon.0) 774 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[79819]: cluster 2024-03-07T03:22:46.524519+0000 mon.a (mon.0) 775 : cluster 0 osdmap e577: 8 total, 8 up, 8 in 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[79819]: cluster 2024-03-07T03:22:47.198843+0000 mon.a (mon.0) 776 : cluster 0 osdmap e578: 8 total, 8 up, 8 in 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[79819]: audit 2024-03-07T03:22:47.201125+0000 mon.c (mon.2) 23 : audit 1 from='client.? 172.21.15.148:0/482266029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[79819]: audit 2024-03-07T03:22:47.211624+0000 mon.a (mon.0) 777 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:47.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:47 smithi148 bash[79819]: audit 2024-03-07T03:22:47.477822+0000 mon.a (mon.0) 778 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:47 smithi195 bash[66542]: cluster 2024-03-07T03:22:46.325406+0000 mgr.y (mgr.34101) 243 : cluster 0 pgmap v460: 137 pgs: 25 unknown, 112 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 20 KiB/s wr, 245 op/s 2024-03-07T03:22:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:47 smithi195 bash[66542]: cluster 2024-03-07T03:22:46.502244+0000 mon.a (mon.0) 774 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:22:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:47 smithi195 bash[66542]: cluster 2024-03-07T03:22:46.524519+0000 mon.a (mon.0) 775 : cluster 0 osdmap e577: 8 total, 8 up, 8 in 2024-03-07T03:22:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:47 smithi195 bash[66542]: cluster 2024-03-07T03:22:47.198843+0000 mon.a (mon.0) 776 : cluster 0 osdmap e578: 8 total, 8 up, 8 in 2024-03-07T03:22:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:47 smithi195 bash[66542]: audit 2024-03-07T03:22:47.201125+0000 mon.c (mon.2) 23 : audit 1 from='client.? 172.21.15.148:0/482266029' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:47 smithi195 bash[66542]: audit 2024-03-07T03:22:47.211624+0000 mon.a (mon.0) 777 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:47.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:47 smithi195 bash[66542]: audit 2024-03-07T03:22:47.477822+0000 mon.a (mon.0) 778 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:48.201 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:49.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:49 smithi148 bash[79819]: audit 2024-03-07T03:22:48.195620+0000 mon.a (mon.0) 779 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:49.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:49 smithi148 bash[79819]: cluster 2024-03-07T03:22:48.203301+0000 mon.a (mon.0) 780 : cluster 0 osdmap e579: 8 total, 8 up, 8 in 2024-03-07T03:22:49.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:49 smithi148 bash[79819]: cluster 2024-03-07T03:22:48.326890+0000 mgr.y (mgr.34101) 244 : cluster 0 pgmap v464: 169 pgs: 2 creating+activating, 6 unknown, 161 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.3 KiB/s wr, 143 op/s 2024-03-07T03:22:49.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:49 smithi148 bash[82222]: audit 2024-03-07T03:22:48.195620+0000 mon.a (mon.0) 779 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:49.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:49 smithi148 bash[82222]: cluster 2024-03-07T03:22:48.203301+0000 mon.a (mon.0) 780 : cluster 0 osdmap e579: 8 total, 8 up, 8 in 2024-03-07T03:22:49.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:49 smithi148 bash[82222]: cluster 2024-03-07T03:22:48.326890+0000 mgr.y (mgr.34101) 244 : cluster 0 pgmap v464: 169 pgs: 2 creating+activating, 6 unknown, 161 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.3 KiB/s wr, 143 op/s 2024-03-07T03:22:49.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:49 smithi195 bash[66542]: audit 2024-03-07T03:22:48.195620+0000 mon.a (mon.0) 779 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:49 smithi195 bash[66542]: cluster 2024-03-07T03:22:48.203301+0000 mon.a (mon.0) 780 : cluster 0 osdmap e579: 8 total, 8 up, 8 in 2024-03-07T03:22:49.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:49 smithi195 bash[66542]: cluster 2024-03-07T03:22:48.326890+0000 mgr.y (mgr.34101) 244 : cluster 0 pgmap v464: 169 pgs: 2 creating+activating, 6 unknown, 161 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.3 KiB/s wr, 143 op/s 2024-03-07T03:22:50.241 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 1 2024-03-07T03:22:50.241 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:22:50.241 INFO:tasks.workunit.client.0.smithi148.stdout:image: image20 2024-03-07T03:22:50.241 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image20 2024-03-07T03:22:50.241 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:50.262 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 2 2024-03-07T03:22:50.262 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:22:50.262 INFO:tasks.workunit.client.0.smithi148.stdout:image: image20 2024-03-07T03:22:50.262 INFO:tasks.workunit.client.0.smithi148.stdout:image: image21 2024-03-07T03:22:50.262 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image20 2024-03-07T03:22:50.262 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image21 2024-03-07T03:22:50.355 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 1 2024-03-07T03:22:50.355 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:22:50.355 INFO:tasks.workunit.client.0.smithi148.stdout:image: image21 2024-03-07T03:22:50.355 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image21 2024-03-07T03:22:50.355 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (3632 ms) 2024-03-07T03:22:50.355 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCopy 2024-03-07T03:22:50.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:50 smithi148 bash[79819]: cluster 2024-03-07T03:22:49.214645+0000 mon.a (mon.0) 781 : cluster 0 osdmap e580: 8 total, 8 up, 8 in 2024-03-07T03:22:50.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:50 smithi148 bash[82222]: cluster 2024-03-07T03:22:49.214645+0000 mon.a (mon.0) 781 : cluster 0 osdmap e580: 8 total, 8 up, 8 in 2024-03-07T03:22:50.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:50 smithi195 bash[66542]: cluster 2024-03-07T03:22:49.214645+0000 mon.a (mon.0) 781 : cluster 0 osdmap e580: 8 total, 8 up, 8 in 2024-03-07T03:22:51.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:51 smithi148 bash[79819]: cluster 2024-03-07T03:22:50.218853+0000 mon.a (mon.0) 782 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2024-03-07T03:22:51.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:51 smithi148 bash[79819]: cluster 2024-03-07T03:22:50.327434+0000 mgr.y (mgr.34101) 245 : cluster 0 pgmap v467: 169 pgs: 2 creating+activating, 3 unknown, 164 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.1 KiB/s wr, 42 op/s 2024-03-07T03:22:51.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:51 smithi148 bash[82222]: cluster 2024-03-07T03:22:50.218853+0000 mon.a (mon.0) 782 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2024-03-07T03:22:51.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:51 smithi148 bash[82222]: cluster 2024-03-07T03:22:50.327434+0000 mgr.y (mgr.34101) 245 : cluster 0 pgmap v467: 169 pgs: 2 creating+activating, 3 unknown, 164 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.1 KiB/s wr, 42 op/s 2024-03-07T03:22:51.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:51 smithi195 bash[66542]: cluster 2024-03-07T03:22:50.218853+0000 mon.a (mon.0) 782 : cluster 0 osdmap e581: 8 total, 8 up, 8 in 2024-03-07T03:22:51.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:51 smithi195 bash[66542]: cluster 2024-03-07T03:22:50.327434+0000 mgr.y (mgr.34101) 245 : cluster 0 pgmap v467: 169 pgs: 2 creating+activating, 3 unknown, 164 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.1 KiB/s wr, 42 op/s 2024-03-07T03:22:52.259 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:22:52.266 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:52.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:52 smithi148 bash[79819]: cluster 2024-03-07T03:22:51.248071+0000 mon.a (mon.0) 783 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2024-03-07T03:22:52.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:52 smithi148 bash[79819]: audit 2024-03-07T03:22:51.254564+0000 mon.c (mon.2) 24 : audit 1 from='client.? 172.21.15.148:0/82250209' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:52.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:52 smithi148 bash[79819]: audit 2024-03-07T03:22:51.256055+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:52.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:52 smithi148 bash[82222]: cluster 2024-03-07T03:22:51.248071+0000 mon.a (mon.0) 783 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2024-03-07T03:22:52.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:52 smithi148 bash[82222]: audit 2024-03-07T03:22:51.254564+0000 mon.c (mon.2) 24 : audit 1 from='client.? 172.21.15.148:0/82250209' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:52.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:52 smithi148 bash[82222]: audit 2024-03-07T03:22:51.256055+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:52.561 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:22:52 smithi195 bash[59134]: ts=2024-03-07T03:22:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:22:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:52 smithi195 bash[66542]: cluster 2024-03-07T03:22:51.248071+0000 mon.a (mon.0) 783 : cluster 0 osdmap e582: 8 total, 8 up, 8 in 2024-03-07T03:22:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:52 smithi195 bash[66542]: audit 2024-03-07T03:22:51.254564+0000 mon.c (mon.2) 24 : audit 1 from='client.? 172.21.15.148:0/82250209' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:52.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:52 smithi195 bash[66542]: audit 2024-03-07T03:22:51.256055+0000 mon.a (mon.0) 784 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:53.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:53 smithi148 bash[79819]: audit 2024-03-07T03:22:52.240017+0000 mon.a (mon.0) 785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:53.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:53 smithi148 bash[79819]: cluster 2024-03-07T03:22:52.246180+0000 mon.a (mon.0) 786 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2024-03-07T03:22:53.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:53 smithi148 bash[79819]: cluster 2024-03-07T03:22:52.327862+0000 mgr.y (mgr.34101) 246 : cluster 0 pgmap v470: 201 pgs: 32 unknown, 2 creating+activating, 167 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.7 KiB/s wr, 9 op/s 2024-03-07T03:22:53.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:53 smithi148 bash[82222]: audit 2024-03-07T03:22:52.240017+0000 mon.a (mon.0) 785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:53.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:53 smithi148 bash[82222]: cluster 2024-03-07T03:22:52.246180+0000 mon.a (mon.0) 786 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2024-03-07T03:22:53.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:53 smithi148 bash[82222]: cluster 2024-03-07T03:22:52.327862+0000 mgr.y (mgr.34101) 246 : cluster 0 pgmap v470: 201 pgs: 32 unknown, 2 creating+activating, 167 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.7 KiB/s wr, 9 op/s 2024-03-07T03:22:53.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:53 smithi195 bash[66542]: audit 2024-03-07T03:22:52.240017+0000 mon.a (mon.0) 785 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:53.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:53 smithi195 bash[66542]: cluster 2024-03-07T03:22:52.246180+0000 mon.a (mon.0) 786 : cluster 0 osdmap e583: 8 total, 8 up, 8 in 2024-03-07T03:22:53.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:53 smithi195 bash[66542]: cluster 2024-03-07T03:22:52.327862+0000 mgr.y (mgr.34101) 246 : cluster 0 pgmap v470: 201 pgs: 32 unknown, 2 creating+activating, 167 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.7 KiB/s wr, 9 op/s 2024-03-07T03:22:54.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:54 smithi148 bash[79819]: cluster 2024-03-07T03:22:53.268581+0000 mon.a (mon.0) 787 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2024-03-07T03:22:54.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:54 smithi148 bash[82222]: cluster 2024-03-07T03:22:53.268581+0000 mon.a (mon.0) 787 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2024-03-07T03:22:54.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:54 smithi195 bash[66542]: cluster 2024-03-07T03:22:53.268581+0000 mon.a (mon.0) 787 : cluster 0 osdmap e584: 8 total, 8 up, 8 in 2024-03-07T03:22:55.217 INFO:tasks.workunit.client.0.smithi148.stdout:image: image22 2024-03-07T03:22:55.217 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image22 2024-03-07T03:22:55.217 INFO:tasks.workunit.client.0.smithi148.stdout:found image22 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:erased image22 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:image: image22 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:image: image23 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image22 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:found image22 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:erased image22 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image23 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:found image23 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:erased image23 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:image: image22 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:image: image23 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:image: image24 2024-03-07T03:22:55.218 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image22 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:found image22 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:erased image22 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image23 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:found image23 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:erased image23 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image24 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:found image24 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:erased image24 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCopy (4863 ms) 2024-03-07T03:22:55.219 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-03-07T03:22:56.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:55 smithi148 bash[79819]: cluster 2024-03-07T03:22:54.282565+0000 mon.a (mon.0) 788 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2024-03-07T03:22:56.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:55 smithi148 bash[79819]: cluster 2024-03-07T03:22:54.328622+0000 mgr.y (mgr.34101) 247 : cluster 0 pgmap v473: 201 pgs: 9 unknown, 192 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-03-07T03:22:56.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:55 smithi148 bash[82222]: cluster 2024-03-07T03:22:54.282565+0000 mon.a (mon.0) 788 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2024-03-07T03:22:56.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:55 smithi148 bash[82222]: cluster 2024-03-07T03:22:54.328622+0000 mgr.y (mgr.34101) 247 : cluster 0 pgmap v473: 201 pgs: 9 unknown, 192 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-03-07T03:22:56.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:55 smithi195 bash[66542]: cluster 2024-03-07T03:22:54.282565+0000 mon.a (mon.0) 788 : cluster 0 osdmap e585: 8 total, 8 up, 8 in 2024-03-07T03:22:56.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:55 smithi195 bash[66542]: cluster 2024-03-07T03:22:54.328622+0000 mgr.y (mgr.34101) 247 : cluster 0 pgmap v473: 201 pgs: 9 unknown, 192 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-03-07T03:22:56.754 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:22:57.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[82222]: audit 2024-03-07T03:22:55.272714+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.148:0/1158599556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:57.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[82222]: cluster 2024-03-07T03:22:55.754010+0000 mon.a (mon.0) 789 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2024-03-07T03:22:57.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[82222]: audit 2024-03-07T03:22:55.757455+0000 mon.c (mon.2) 26 : audit 1 from='client.? 172.21.15.148:0/2268811953' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:57.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[82222]: audit 2024-03-07T03:22:55.759481+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:57.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[82222]: cluster 2024-03-07T03:22:56.329749+0000 mgr.y (mgr.34101) 248 : cluster 0 pgmap v475: 233 pgs: 5 creating+peering, 27 unknown, 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.9 KiB/s wr, 59 op/s 2024-03-07T03:22:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[79819]: audit 2024-03-07T03:22:55.272714+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.148:0/1158599556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[79819]: cluster 2024-03-07T03:22:55.754010+0000 mon.a (mon.0) 789 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2024-03-07T03:22:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[79819]: audit 2024-03-07T03:22:55.757455+0000 mon.c (mon.2) 26 : audit 1 from='client.? 172.21.15.148:0/2268811953' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[79819]: audit 2024-03-07T03:22:55.759481+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:56 smithi148 bash[79819]: cluster 2024-03-07T03:22:56.329749+0000 mgr.y (mgr.34101) 248 : cluster 0 pgmap v475: 233 pgs: 5 creating+peering, 27 unknown, 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.9 KiB/s wr, 59 op/s 2024-03-07T03:22:57.061 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:56 smithi195 bash[66542]: audit 2024-03-07T03:22:55.272714+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.148:0/1158599556' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:57.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:56 smithi195 bash[66542]: cluster 2024-03-07T03:22:55.754010+0000 mon.a (mon.0) 789 : cluster 0 osdmap e586: 8 total, 8 up, 8 in 2024-03-07T03:22:57.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:56 smithi195 bash[66542]: audit 2024-03-07T03:22:55.757455+0000 mon.c (mon.2) 26 : audit 1 from='client.? 172.21.15.148:0/2268811953' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:57.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:56 smithi195 bash[66542]: audit 2024-03-07T03:22:55.759481+0000 mon.a (mon.0) 790 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:22:57.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:56 smithi195 bash[66542]: cluster 2024-03-07T03:22:56.329749+0000 mgr.y (mgr.34101) 248 : cluster 0 pgmap v475: 233 pgs: 5 creating+peering, 27 unknown, 201 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.9 KiB/s wr, 59 op/s 2024-03-07T03:22:57.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:22:57] "GET /metrics HTTP/1.1" 200 36460 "" "Prometheus/2.43.0" 2024-03-07T03:22:58.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[79819]: cluster 2024-03-07T03:22:56.745239+0000 mon.a (mon.0) 791 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:58.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[79819]: audit 2024-03-07T03:22:56.754349+0000 mon.a (mon.0) 792 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:58.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[79819]: cluster 2024-03-07T03:22:56.761609+0000 mon.a (mon.0) 793 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[79819]: cluster 2024-03-07T03:22:57.204106+0000 mon.a (mon.0) 794 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[79819]: audit 2024-03-07T03:22:57.317142+0000 mon.a (mon.0) 795 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[79819]: audit 2024-03-07T03:22:57.318746+0000 mon.a (mon.0) 796 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[79819]: audit 2024-03-07T03:22:57.631394+0000 mon.a (mon.0) 797 : audit 0 from='client.? 172.21.15.148:0/3719123074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[82222]: cluster 2024-03-07T03:22:56.745239+0000 mon.a (mon.0) 791 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[82222]: audit 2024-03-07T03:22:56.754349+0000 mon.a (mon.0) 792 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[82222]: cluster 2024-03-07T03:22:56.761609+0000 mon.a (mon.0) 793 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[82222]: cluster 2024-03-07T03:22:57.204106+0000 mon.a (mon.0) 794 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[82222]: audit 2024-03-07T03:22:57.317142+0000 mon.a (mon.0) 795 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[82222]: audit 2024-03-07T03:22:57.318746+0000 mon.a (mon.0) 796 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:57 smithi148 bash[82222]: audit 2024-03-07T03:22:57.631394+0000 mon.a (mon.0) 797 : audit 0 from='client.? 172.21.15.148:0/3719123074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:58.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:57 smithi195 bash[66542]: cluster 2024-03-07T03:22:56.745239+0000 mon.a (mon.0) 791 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:22:58.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:57 smithi195 bash[66542]: audit 2024-03-07T03:22:56.754349+0000 mon.a (mon.0) 792 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:22:58.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:57 smithi195 bash[66542]: cluster 2024-03-07T03:22:56.761609+0000 mon.a (mon.0) 793 : cluster 0 osdmap e587: 8 total, 8 up, 8 in 2024-03-07T03:22:58.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:57 smithi195 bash[66542]: cluster 2024-03-07T03:22:57.204106+0000 mon.a (mon.0) 794 : cluster 0 osdmap e588: 8 total, 8 up, 8 in 2024-03-07T03:22:58.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:57 smithi195 bash[66542]: audit 2024-03-07T03:22:57.317142+0000 mon.a (mon.0) 795 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:22:58.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:57 smithi195 bash[66542]: audit 2024-03-07T03:22:57.318746+0000 mon.a (mon.0) 796 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:22:58.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:57 smithi195 bash[66542]: audit 2024-03-07T03:22:57.631394+0000 mon.a (mon.0) 797 : audit 0 from='client.? 172.21.15.148:0/3719123074' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:22:58.848 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 1 2024-03-07T03:22:58.849 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:22:58.849 INFO:tasks.workunit.client.0.smithi148.stdout:image: image25 2024-03-07T03:22:58.849 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image25 2024-03-07T03:22:58.898 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 2 2024-03-07T03:22:58.898 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:22:58.898 INFO:tasks.workunit.client.0.smithi148.stdout:image: image25 2024-03-07T03:22:58.898 INFO:tasks.workunit.client.0.smithi148.stdout:image: image26 2024-03-07T03:22:58.898 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image25 2024-03-07T03:22:58.898 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image26 2024-03-07T03:22:59.031 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 3 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:image: image25 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:image: image26 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:image: image27 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image25 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image26 2024-03-07T03:22:59.032 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image27 2024-03-07T03:22:59.152 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCopyPP (3935 ms) 2024-03-07T03:22:59.153 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-03-07T03:22:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:59 smithi148 bash[79819]: cluster 2024-03-07T03:22:58.207526+0000 mon.a (mon.0) 798 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2024-03-07T03:22:59.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:22:59 smithi148 bash[79819]: cluster 2024-03-07T03:22:58.331824+0000 mgr.y (mgr.34101) 249 : cluster 0 pgmap v479: 233 pgs: 5 creating+peering, 7 unknown, 221 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 342 KiB/s rd, 22 KiB/s wr, 399 op/s 2024-03-07T03:22:59.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:59 smithi148 bash[82222]: cluster 2024-03-07T03:22:58.207526+0000 mon.a (mon.0) 798 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2024-03-07T03:22:59.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:22:59 smithi148 bash[82222]: cluster 2024-03-07T03:22:58.331824+0000 mgr.y (mgr.34101) 249 : cluster 0 pgmap v479: 233 pgs: 5 creating+peering, 7 unknown, 221 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 342 KiB/s rd, 22 KiB/s wr, 399 op/s 2024-03-07T03:22:59.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:59 smithi195 bash[66542]: cluster 2024-03-07T03:22:58.207526+0000 mon.a (mon.0) 798 : cluster 0 osdmap e589: 8 total, 8 up, 8 in 2024-03-07T03:22:59.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:22:59 smithi195 bash[66542]: cluster 2024-03-07T03:22:58.331824+0000 mgr.y (mgr.34101) 249 : cluster 0 pgmap v479: 233 pgs: 5 creating+peering, 7 unknown, 221 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 342 KiB/s rd, 22 KiB/s wr, 399 op/s 2024-03-07T03:23:00.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:00 smithi148 bash[82222]: cluster 2024-03-07T03:22:59.200879+0000 mon.a (mon.0) 799 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:23:00.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:00 smithi148 bash[82222]: audit 2024-03-07T03:22:59.978829+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.148:0/1219773482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:00.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:00 smithi148 bash[79819]: cluster 2024-03-07T03:22:59.200879+0000 mon.a (mon.0) 799 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:23:00.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:00 smithi148 bash[79819]: audit 2024-03-07T03:22:59.978829+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.148:0/1219773482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:00.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:00 smithi148 bash[72584]: debug 2024-03-07T03:23:00.407+0000 7f6cc280d700 -1 mgr.server reply reply (11) Resource temporarily unavailable 29 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:23:00.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:00 smithi148 bash[72584]: debug 2024-03-07T03:23:00.407+0000 7f6cc280d700 -1 mgr.server reply reply (16) Device or resource busy 29 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-07T03:23:00.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:00 smithi195 bash[66542]: cluster 2024-03-07T03:22:59.200879+0000 mon.a (mon.0) 799 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:23:00.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:00 smithi195 bash[66542]: audit 2024-03-07T03:22:59.978829+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.148:0/1219773482' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:01.240 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:01.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[79819]: cluster 2024-03-07T03:23:00.238942+0000 mon.a (mon.0) 800 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2024-03-07T03:23:01.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[79819]: audit 2024-03-07T03:23:00.247173+0000 mon.c (mon.2) 28 : audit 1 from='client.? 172.21.15.148:0/2084634754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:01.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[79819]: audit 2024-03-07T03:23:00.250428+0000 mon.a (mon.0) 801 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:01.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[79819]: cluster 2024-03-07T03:23:00.333068+0000 mgr.y (mgr.34101) 250 : cluster 0 pgmap v481: 265 pgs: 8 creating+peering, 29 unknown, 228 active+clean; 583 KiB data, 451 MiB used, 715 GiB / 715 GiB avail; 339 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[79819]: audit 2024-03-07T03:23:00.408440+0000 mon.a (mon.0) 802 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[79819]: audit 2024-03-07T03:23:00.408888+0000 mgr.y (mgr.34101) 251 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[79819]: cephadm 2024-03-07T03:23:00.411202+0000 mgr.y (mgr.34101) 252 : cephadm 1 Upgrade: 29 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[82222]: cluster 2024-03-07T03:23:00.238942+0000 mon.a (mon.0) 800 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[82222]: audit 2024-03-07T03:23:00.247173+0000 mon.c (mon.2) 28 : audit 1 from='client.? 172.21.15.148:0/2084634754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[82222]: audit 2024-03-07T03:23:00.250428+0000 mon.a (mon.0) 801 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[82222]: cluster 2024-03-07T03:23:00.333068+0000 mgr.y (mgr.34101) 250 : cluster 0 pgmap v481: 265 pgs: 8 creating+peering, 29 unknown, 228 active+clean; 583 KiB data, 451 MiB used, 715 GiB / 715 GiB avail; 339 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[82222]: audit 2024-03-07T03:23:00.408440+0000 mon.a (mon.0) 802 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[82222]: audit 2024-03-07T03:23:00.408888+0000 mgr.y (mgr.34101) 251 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:01 smithi148 bash[82222]: cephadm 2024-03-07T03:23:00.411202+0000 mgr.y (mgr.34101) 252 : cephadm 1 Upgrade: 29 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:23:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:01 smithi195 bash[66542]: cluster 2024-03-07T03:23:00.238942+0000 mon.a (mon.0) 800 : cluster 0 osdmap e590: 8 total, 8 up, 8 in 2024-03-07T03:23:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:01 smithi195 bash[66542]: audit 2024-03-07T03:23:00.247173+0000 mon.c (mon.2) 28 : audit 1 from='client.? 172.21.15.148:0/2084634754' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:01 smithi195 bash[66542]: audit 2024-03-07T03:23:00.250428+0000 mon.a (mon.0) 801 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:01 smithi195 bash[66542]: cluster 2024-03-07T03:23:00.333068+0000 mgr.y (mgr.34101) 250 : cluster 0 pgmap v481: 265 pgs: 8 creating+peering, 29 unknown, 228 active+clean; 583 KiB data, 451 MiB used, 715 GiB / 715 GiB avail; 339 KiB/s rd, 21 KiB/s wr, 392 op/s 2024-03-07T03:23:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:01 smithi195 bash[66542]: audit 2024-03-07T03:23:00.408440+0000 mon.a (mon.0) 802 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:01 smithi195 bash[66542]: audit 2024-03-07T03:23:00.408888+0000 mgr.y (mgr.34101) 251 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:01.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:01 smithi195 bash[66542]: cephadm 2024-03-07T03:23:00.411202+0000 mgr.y (mgr.34101) 252 : cephadm 1 Upgrade: 29 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[79819]: cluster 2024-03-07T03:23:01.227701+0000 mon.a (mon.0) 803 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[79819]: audit 2024-03-07T03:23:01.239048+0000 mon.a (mon.0) 804 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[79819]: cluster 2024-03-07T03:23:01.249555+0000 mon.a (mon.0) 805 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[79819]: cluster 2024-03-07T03:23:02.204256+0000 mon.a (mon.0) 806 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[82222]: cluster 2024-03-07T03:23:01.227701+0000 mon.a (mon.0) 803 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[82222]: audit 2024-03-07T03:23:01.239048+0000 mon.a (mon.0) 804 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[82222]: cluster 2024-03-07T03:23:01.249555+0000 mon.a (mon.0) 805 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2024-03-07T03:23:02.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:02 smithi148 bash[82222]: cluster 2024-03-07T03:23:02.204256+0000 mon.a (mon.0) 806 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2024-03-07T03:23:02.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:23:02 smithi195 bash[59134]: ts=2024-03-07T03:23:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:02 smithi195 bash[66542]: cluster 2024-03-07T03:23:01.227701+0000 mon.a (mon.0) 803 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:23:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:02 smithi195 bash[66542]: audit 2024-03-07T03:23:01.239048+0000 mon.a (mon.0) 804 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:02 smithi195 bash[66542]: cluster 2024-03-07T03:23:01.249555+0000 mon.a (mon.0) 805 : cluster 0 osdmap e591: 8 total, 8 up, 8 in 2024-03-07T03:23:02.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:02 smithi195 bash[66542]: cluster 2024-03-07T03:23:02.204256+0000 mon.a (mon.0) 806 : cluster 0 osdmap e592: 8 total, 8 up, 8 in 2024-03-07T03:23:03.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:03 smithi148 bash[79819]: cluster 2024-03-07T03:23:02.333880+0000 mgr.y (mgr.34101) 253 : cluster 0 pgmap v484: 265 pgs: 8 creating+peering, 24 unknown, 233 active+clean; 583 KiB data, 451 MiB used, 715 GiB / 715 GiB avail; 329 KiB/s rd, 21 KiB/s wr, 384 op/s 2024-03-07T03:23:03.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:03 smithi148 bash[79819]: audit 2024-03-07T03:23:02.349296+0000 mon.a (mon.0) 807 : audit 0 from='client.? 172.21.15.148:0/192176254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:03.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:03 smithi148 bash[79819]: cluster 2024-03-07T03:23:03.208300+0000 mon.a (mon.0) 808 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2024-03-07T03:23:03.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:03 smithi148 bash[82222]: cluster 2024-03-07T03:23:02.333880+0000 mgr.y (mgr.34101) 253 : cluster 0 pgmap v484: 265 pgs: 8 creating+peering, 24 unknown, 233 active+clean; 583 KiB data, 451 MiB used, 715 GiB / 715 GiB avail; 329 KiB/s rd, 21 KiB/s wr, 384 op/s 2024-03-07T03:23:03.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:03 smithi148 bash[82222]: audit 2024-03-07T03:23:02.349296+0000 mon.a (mon.0) 807 : audit 0 from='client.? 172.21.15.148:0/192176254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:03.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:03 smithi148 bash[82222]: cluster 2024-03-07T03:23:03.208300+0000 mon.a (mon.0) 808 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2024-03-07T03:23:03.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:03 smithi195 bash[66542]: cluster 2024-03-07T03:23:02.333880+0000 mgr.y (mgr.34101) 253 : cluster 0 pgmap v484: 265 pgs: 8 creating+peering, 24 unknown, 233 active+clean; 583 KiB data, 451 MiB used, 715 GiB / 715 GiB avail; 329 KiB/s rd, 21 KiB/s wr, 384 op/s 2024-03-07T03:23:03.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:03 smithi195 bash[66542]: audit 2024-03-07T03:23:02.349296+0000 mon.a (mon.0) 807 : audit 0 from='client.? 172.21.15.148:0/192176254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:03.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:03 smithi195 bash[66542]: cluster 2024-03-07T03:23:03.208300+0000 mon.a (mon.0) 808 : cluster 0 osdmap e593: 8 total, 8 up, 8 in 2024-03-07T03:23:04.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:04 smithi148 bash[79819]: cluster 2024-03-07T03:23:03.240957+0000 mon.a (mon.0) 809 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:23:04.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:04 smithi148 bash[82222]: cluster 2024-03-07T03:23:03.240957+0000 mon.a (mon.0) 809 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:23:04.561 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:04 smithi195 bash[66542]: cluster 2024-03-07T03:23:03.240957+0000 mon.a (mon.0) 809 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:23:05.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:05 smithi148 bash[79819]: cluster 2024-03-07T03:23:04.335988+0000 mgr.y (mgr.34101) 254 : cluster 0 pgmap v486: 265 pgs: 8 creating+peering, 1 active+clean+snaptrim_wait, 256 active+clean; 583 KiB data, 463 MiB used, 715 GiB / 715 GiB avail; 333 KiB/s rd, 22 KiB/s wr, 389 op/s 2024-03-07T03:23:05.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:05 smithi148 bash[79819]: audit 2024-03-07T03:23:04.702962+0000 mon.a (mon.0) 810 : audit 0 from='client.? 172.21.15.148:0/4223197613' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:05.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:05 smithi148 bash[82222]: cluster 2024-03-07T03:23:04.335988+0000 mgr.y (mgr.34101) 254 : cluster 0 pgmap v486: 265 pgs: 8 creating+peering, 1 active+clean+snaptrim_wait, 256 active+clean; 583 KiB data, 463 MiB used, 715 GiB / 715 GiB avail; 333 KiB/s rd, 22 KiB/s wr, 389 op/s 2024-03-07T03:23:05.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:05 smithi148 bash[82222]: audit 2024-03-07T03:23:04.702962+0000 mon.a (mon.0) 810 : audit 0 from='client.? 172.21.15.148:0/4223197613' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:05 smithi195 bash[66542]: cluster 2024-03-07T03:23:04.335988+0000 mgr.y (mgr.34101) 254 : cluster 0 pgmap v486: 265 pgs: 8 creating+peering, 1 active+clean+snaptrim_wait, 256 active+clean; 583 KiB data, 463 MiB used, 715 GiB / 715 GiB avail; 333 KiB/s rd, 22 KiB/s wr, 389 op/s 2024-03-07T03:23:05.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:05 smithi195 bash[66542]: audit 2024-03-07T03:23:04.702962+0000 mon.a (mon.0) 810 : audit 0 from='client.? 172.21.15.148:0/4223197613' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:06.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:06 smithi148 bash[79819]: cluster 2024-03-07T03:23:05.269880+0000 mon.a (mon.0) 811 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2024-03-07T03:23:06.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:06 smithi148 bash[82222]: cluster 2024-03-07T03:23:05.269880+0000 mon.a (mon.0) 811 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2024-03-07T03:23:06.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:06 smithi195 bash[66542]: cluster 2024-03-07T03:23:05.269880+0000 mon.a (mon.0) 811 : cluster 0 osdmap e594: 8 total, 8 up, 8 in 2024-03-07T03:23:07.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:23:07] "GET /metrics HTTP/1.1" 200 46069 "" "Prometheus/2.43.0" 2024-03-07T03:23:07.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[79819]: cluster 2024-03-07T03:23:06.273188+0000 mon.a (mon.0) 812 : cluster 0 osdmap e595: 8 total, 8 up, 8 in 2024-03-07T03:23:07.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[79819]: cluster 2024-03-07T03:23:06.336840+0000 mgr.y (mgr.34101) 255 : cluster 0 pgmap v489: 265 pgs: 1 active+clean+snaptrim_wait, 264 active+clean; 583 KiB data, 463 MiB used, 715 GiB / 715 GiB avail; 322 KiB/s rd, 22 KiB/s wr, 380 op/s 2024-03-07T03:23:07.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[79819]: audit 2024-03-07T03:23:07.053842+0000 mon.a (mon.0) 813 : audit 0 from='client.? 172.21.15.148:0/3896826298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:07.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[79819]: cluster 2024-03-07T03:23:07.208990+0000 mon.a (mon.0) 814 : cluster 0 osdmap e596: 8 total, 8 up, 8 in 2024-03-07T03:23:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[82222]: cluster 2024-03-07T03:23:06.273188+0000 mon.a (mon.0) 812 : cluster 0 osdmap e595: 8 total, 8 up, 8 in 2024-03-07T03:23:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[82222]: cluster 2024-03-07T03:23:06.336840+0000 mgr.y (mgr.34101) 255 : cluster 0 pgmap v489: 265 pgs: 1 active+clean+snaptrim_wait, 264 active+clean; 583 KiB data, 463 MiB used, 715 GiB / 715 GiB avail; 322 KiB/s rd, 22 KiB/s wr, 380 op/s 2024-03-07T03:23:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[82222]: audit 2024-03-07T03:23:07.053842+0000 mon.a (mon.0) 813 : audit 0 from='client.? 172.21.15.148:0/3896826298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:07.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:07 smithi148 bash[82222]: cluster 2024-03-07T03:23:07.208990+0000 mon.a (mon.0) 814 : cluster 0 osdmap e596: 8 total, 8 up, 8 in 2024-03-07T03:23:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:07 smithi195 bash[66542]: cluster 2024-03-07T03:23:06.273188+0000 mon.a (mon.0) 812 : cluster 0 osdmap e595: 8 total, 8 up, 8 in 2024-03-07T03:23:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:07 smithi195 bash[66542]: cluster 2024-03-07T03:23:06.336840+0000 mgr.y (mgr.34101) 255 : cluster 0 pgmap v489: 265 pgs: 1 active+clean+snaptrim_wait, 264 active+clean; 583 KiB data, 463 MiB used, 715 GiB / 715 GiB avail; 322 KiB/s rd, 22 KiB/s wr, 380 op/s 2024-03-07T03:23:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:07 smithi195 bash[66542]: audit 2024-03-07T03:23:07.053842+0000 mon.a (mon.0) 813 : audit 0 from='client.? 172.21.15.148:0/3896826298' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:07.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:07 smithi195 bash[66542]: cluster 2024-03-07T03:23:07.208990+0000 mon.a (mon.0) 814 : cluster 0 osdmap e596: 8 total, 8 up, 8 in 2024-03-07T03:23:08.473 INFO:tasks.workunit.client.0.smithi148.stdout:image: image28 2024-03-07T03:23:08.473 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image28 2024-03-07T03:23:08.473 INFO:tasks.workunit.client.0.smithi148.stdout:found image28 2024-03-07T03:23:08.473 INFO:tasks.workunit.client.0.smithi148.stdout:erased image28 2024-03-07T03:23:08.473 INFO:tasks.workunit.client.0.smithi148.stdout:image: image28 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:image: image29 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image28 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:found image28 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:erased image28 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image29 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:found image29 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:erased image29 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:image: image28 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:image: image29 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:image: image30 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image28 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:found image28 2024-03-07T03:23:08.474 INFO:tasks.workunit.client.0.smithi148.stdout:erased image28 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image29 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:found image29 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:erased image29 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image30 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:found image30 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:erased image30 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:image: image28 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:image: image29 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:image: image30 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:image: image31 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image28 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:found image28 2024-03-07T03:23:08.475 INFO:tasks.workunit.client.0.smithi148.stdout:erased image28 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image29 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:found image29 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:erased image29 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image30 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:found image30 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:erased image30 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image31 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:found image31 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:erased image31 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:image: image28 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:image: image29 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:image: image30 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:image: image31 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:image: image32 2024-03-07T03:23:08.476 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image28 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:found image28 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:erased image28 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image29 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:found image29 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:erased image29 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image30 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:found image30 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:erased image30 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image31 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:found image31 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:erased image31 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image32 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:found image32 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:erased image32 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:image: image28 2024-03-07T03:23:08.477 INFO:tasks.workunit.client.0.smithi148.stdout:image: image29 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:image: image30 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:image: image31 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:image: image32 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:image: image33 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image28 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:found image28 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:erased image28 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image29 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:found image29 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:erased image29 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image30 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:found image30 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:erased image30 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image31 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:found image31 2024-03-07T03:23:08.478 INFO:tasks.workunit.client.0.smithi148.stdout:erased image31 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image32 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:found image32 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:erased image32 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image33 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:found image33 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:erased image33 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestDeepCopy (9321 ms) 2024-03-07T03:23:08.479 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-03-07T03:23:09.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:09 smithi148 bash[79819]: cluster 2024-03-07T03:23:08.295742+0000 mon.a (mon.0) 815 : cluster 0 osdmap e597: 8 total, 8 up, 8 in 2024-03-07T03:23:09.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:09 smithi148 bash[79819]: cluster 2024-03-07T03:23:08.337888+0000 mgr.y (mgr.34101) 256 : cluster 0 pgmap v492: 265 pgs: 265 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 505 KiB/s rd, 29 KiB/s wr, 597 op/s 2024-03-07T03:23:09.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:09 smithi148 bash[82222]: cluster 2024-03-07T03:23:08.295742+0000 mon.a (mon.0) 815 : cluster 0 osdmap e597: 8 total, 8 up, 8 in 2024-03-07T03:23:09.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:09 smithi148 bash[82222]: cluster 2024-03-07T03:23:08.337888+0000 mgr.y (mgr.34101) 256 : cluster 0 pgmap v492: 265 pgs: 265 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 505 KiB/s rd, 29 KiB/s wr, 597 op/s 2024-03-07T03:23:09.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:09 smithi195 bash[66542]: cluster 2024-03-07T03:23:08.295742+0000 mon.a (mon.0) 815 : cluster 0 osdmap e597: 8 total, 8 up, 8 in 2024-03-07T03:23:09.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:09 smithi195 bash[66542]: cluster 2024-03-07T03:23:08.337888+0000 mgr.y (mgr.34101) 256 : cluster 0 pgmap v492: 265 pgs: 265 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 505 KiB/s rd, 29 KiB/s wr, 597 op/s 2024-03-07T03:23:10.319 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:10.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:10 smithi195 bash[66542]: cluster 2024-03-07T03:23:09.310312+0000 mon.a (mon.0) 816 : cluster 0 osdmap e598: 8 total, 8 up, 8 in 2024-03-07T03:23:10.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:10 smithi195 bash[66542]: audit 2024-03-07T03:23:09.314096+0000 mon.a (mon.0) 817 : audit 1 from='client.? 172.21.15.148:0/724790609' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:10.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:10 smithi195 bash[66542]: audit 2024-03-07T03:23:09.421325+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.148:0/1910246875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:10.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:10 smithi148 bash[79819]: cluster 2024-03-07T03:23:09.310312+0000 mon.a (mon.0) 816 : cluster 0 osdmap e598: 8 total, 8 up, 8 in 2024-03-07T03:23:10.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:10 smithi148 bash[79819]: audit 2024-03-07T03:23:09.314096+0000 mon.a (mon.0) 817 : audit 1 from='client.? 172.21.15.148:0/724790609' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:10.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:10 smithi148 bash[79819]: audit 2024-03-07T03:23:09.421325+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.148:0/1910246875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:10.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:10 smithi148 bash[82222]: cluster 2024-03-07T03:23:09.310312+0000 mon.a (mon.0) 816 : cluster 0 osdmap e598: 8 total, 8 up, 8 in 2024-03-07T03:23:10.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:10 smithi148 bash[82222]: audit 2024-03-07T03:23:09.314096+0000 mon.a (mon.0) 817 : audit 1 from='client.? 172.21.15.148:0/724790609' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-21","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:10.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:10 smithi148 bash[82222]: audit 2024-03-07T03:23:09.421325+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.148:0/1910246875' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:11 smithi148 bash[82222]: audit 2024-03-07T03:23:10.303792+0000 mon.a (mon.0) 818 : audit 1 from='client.? 172.21.15.148:0/724790609' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:11 smithi148 bash[82222]: cluster 2024-03-07T03:23:10.311802+0000 mon.a (mon.0) 819 : cluster 0 osdmap e599: 8 total, 8 up, 8 in 2024-03-07T03:23:11.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:11 smithi148 bash[82222]: cluster 2024-03-07T03:23:10.338505+0000 mgr.y (mgr.34101) 257 : cluster 0 pgmap v495: 297 pgs: 8 creating+peering, 24 unknown, 265 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 494 KiB/s rd, 28 KiB/s wr, 583 op/s 2024-03-07T03:23:11.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:11 smithi148 bash[79819]: audit 2024-03-07T03:23:10.303792+0000 mon.a (mon.0) 818 : audit 1 from='client.? 172.21.15.148:0/724790609' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:11.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:11 smithi148 bash[79819]: cluster 2024-03-07T03:23:10.311802+0000 mon.a (mon.0) 819 : cluster 0 osdmap e599: 8 total, 8 up, 8 in 2024-03-07T03:23:11.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:11 smithi148 bash[79819]: cluster 2024-03-07T03:23:10.338505+0000 mgr.y (mgr.34101) 257 : cluster 0 pgmap v495: 297 pgs: 8 creating+peering, 24 unknown, 265 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 494 KiB/s rd, 28 KiB/s wr, 583 op/s 2024-03-07T03:23:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:11 smithi195 bash[66542]: audit 2024-03-07T03:23:10.303792+0000 mon.a (mon.0) 818 : audit 1 from='client.? 172.21.15.148:0/724790609' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:11 smithi195 bash[66542]: cluster 2024-03-07T03:23:10.311802+0000 mon.a (mon.0) 819 : cluster 0 osdmap e599: 8 total, 8 up, 8 in 2024-03-07T03:23:11.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:11 smithi195 bash[66542]: cluster 2024-03-07T03:23:10.338505+0000 mgr.y (mgr.34101) 257 : cluster 0 pgmap v495: 297 pgs: 8 creating+peering, 24 unknown, 265 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 494 KiB/s rd, 28 KiB/s wr, 583 op/s 2024-03-07T03:23:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:23:12 smithi195 bash[59134]: ts=2024-03-07T03:23:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:12 smithi195 bash[66542]: cluster 2024-03-07T03:23:11.345990+0000 mon.a (mon.0) 820 : cluster 0 osdmap e600: 8 total, 8 up, 8 in 2024-03-07T03:23:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:12 smithi195 bash[66542]: audit 2024-03-07T03:23:11.772186+0000 mon.a (mon.0) 821 : audit 0 from='client.? 172.21.15.148:0/3999333023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:12.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:12 smithi195 bash[66542]: cluster 2024-03-07T03:23:12.212190+0000 mon.a (mon.0) 822 : cluster 0 osdmap e601: 8 total, 8 up, 8 in 2024-03-07T03:23:12.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:12 smithi195 bash[66542]: audit 2024-03-07T03:23:12.319024+0000 mon.a (mon.0) 823 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:12.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:12 smithi195 bash[66542]: audit 2024-03-07T03:23:12.321578+0000 mon.a (mon.0) 824 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:12.674 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 1 2024-03-07T03:23:12.674 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:23:12.674 INFO:tasks.workunit.client.0.smithi148.stdout:image: image34 2024-03-07T03:23:12.674 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image34 2024-03-07T03:23:12.735 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 2 2024-03-07T03:23:12.735 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:23:12.735 INFO:tasks.workunit.client.0.smithi148.stdout:image: image34 2024-03-07T03:23:12.735 INFO:tasks.workunit.client.0.smithi148.stdout:image: image35 2024-03-07T03:23:12.736 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image34 2024-03-07T03:23:12.736 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image35 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[79819]: cluster 2024-03-07T03:23:11.345990+0000 mon.a (mon.0) 820 : cluster 0 osdmap e600: 8 total, 8 up, 8 in 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[79819]: audit 2024-03-07T03:23:11.772186+0000 mon.a (mon.0) 821 : audit 0 from='client.? 172.21.15.148:0/3999333023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[79819]: cluster 2024-03-07T03:23:12.212190+0000 mon.a (mon.0) 822 : cluster 0 osdmap e601: 8 total, 8 up, 8 in 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[79819]: audit 2024-03-07T03:23:12.319024+0000 mon.a (mon.0) 823 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[79819]: audit 2024-03-07T03:23:12.321578+0000 mon.a (mon.0) 824 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[82222]: cluster 2024-03-07T03:23:11.345990+0000 mon.a (mon.0) 820 : cluster 0 osdmap e600: 8 total, 8 up, 8 in 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[82222]: audit 2024-03-07T03:23:11.772186+0000 mon.a (mon.0) 821 : audit 0 from='client.? 172.21.15.148:0/3999333023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[82222]: cluster 2024-03-07T03:23:12.212190+0000 mon.a (mon.0) 822 : cluster 0 osdmap e601: 8 total, 8 up, 8 in 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[82222]: audit 2024-03-07T03:23:12.319024+0000 mon.a (mon.0) 823 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:12.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:12 smithi148 bash[82222]: audit 2024-03-07T03:23:12.321578+0000 mon.a (mon.0) 824 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:num images is: 3 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:image: image34 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:image: image35 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:image: image36 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image34 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image35 2024-03-07T03:23:12.852 INFO:tasks.workunit.client.0.smithi148.stdout:expected = image36 2024-03-07T03:23:12.945 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4472 ms) 2024-03-07T03:23:12.945 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-03-07T03:23:12.945 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:13 smithi148 bash[82222]: cluster 2024-03-07T03:23:12.339285+0000 mgr.y (mgr.34101) 258 : cluster 0 pgmap v498: 297 pgs: 8 creating+peering, 22 unknown, 267 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 9 op/s 2024-03-07T03:23:13.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:13 smithi148 bash[82222]: audit 2024-03-07T03:23:12.504669+0000 mon.a (mon.0) 825 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:13 smithi148 bash[79819]: cluster 2024-03-07T03:23:12.339285+0000 mgr.y (mgr.34101) 258 : cluster 0 pgmap v498: 297 pgs: 8 creating+peering, 22 unknown, 267 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 9 op/s 2024-03-07T03:23:13.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:13 smithi148 bash[79819]: audit 2024-03-07T03:23:12.504669+0000 mon.a (mon.0) 825 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:13 smithi195 bash[66542]: cluster 2024-03-07T03:23:12.339285+0000 mgr.y (mgr.34101) 258 : cluster 0 pgmap v498: 297 pgs: 8 creating+peering, 22 unknown, 267 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1023 B/s wr, 9 op/s 2024-03-07T03:23:13.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:13 smithi195 bash[66542]: audit 2024-03-07T03:23:12.504669+0000 mon.a (mon.0) 825 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:14 smithi148 bash[79819]: cluster 2024-03-07T03:23:13.528146+0000 mon.a (mon.0) 826 : cluster 0 osdmap e602: 8 total, 8 up, 8 in 2024-03-07T03:23:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:14 smithi148 bash[79819]: audit 2024-03-07T03:23:14.127434+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.148:0/1567546687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:14.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:14 smithi148 bash[82222]: cluster 2024-03-07T03:23:13.528146+0000 mon.a (mon.0) 826 : cluster 0 osdmap e602: 8 total, 8 up, 8 in 2024-03-07T03:23:14.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:14 smithi148 bash[82222]: audit 2024-03-07T03:23:14.127434+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.148:0/1567546687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:14.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:14 smithi195 bash[66542]: cluster 2024-03-07T03:23:13.528146+0000 mon.a (mon.0) 826 : cluster 0 osdmap e602: 8 total, 8 up, 8 in 2024-03-07T03:23:14.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:14 smithi195 bash[66542]: audit 2024-03-07T03:23:14.127434+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.148:0/1567546687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:15.789 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:15 smithi148 bash[72584]: debug 2024-03-07T03:23:15.422+0000 7f6cc280d700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-07T03:23:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:15 smithi148 bash[79819]: cluster 2024-03-07T03:23:14.340107+0000 mgr.y (mgr.34101) 259 : cluster 0 pgmap v500: 297 pgs: 8 creating+peering, 289 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 369 KiB/s rd, 24 KiB/s wr, 436 op/s 2024-03-07T03:23:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:15 smithi148 bash[79819]: cluster 2024-03-07T03:23:14.540888+0000 mon.a (mon.0) 827 : cluster 0 osdmap e603: 8 total, 8 up, 8 in 2024-03-07T03:23:15.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:15 smithi148 bash[79819]: audit 2024-03-07T03:23:15.425814+0000 mon.a (mon.0) 828 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:15 smithi148 bash[82222]: cluster 2024-03-07T03:23:14.340107+0000 mgr.y (mgr.34101) 259 : cluster 0 pgmap v500: 297 pgs: 8 creating+peering, 289 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 369 KiB/s rd, 24 KiB/s wr, 436 op/s 2024-03-07T03:23:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:15 smithi148 bash[82222]: cluster 2024-03-07T03:23:14.540888+0000 mon.a (mon.0) 827 : cluster 0 osdmap e603: 8 total, 8 up, 8 in 2024-03-07T03:23:15.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:15 smithi148 bash[82222]: audit 2024-03-07T03:23:15.425814+0000 mon.a (mon.0) 828 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:15.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:15 smithi195 bash[66542]: cluster 2024-03-07T03:23:14.340107+0000 mgr.y (mgr.34101) 259 : cluster 0 pgmap v500: 297 pgs: 8 creating+peering, 289 active+clean; 584 KiB data, 464 MiB used, 715 GiB / 715 GiB avail; 369 KiB/s rd, 24 KiB/s wr, 436 op/s 2024-03-07T03:23:15.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:15 smithi195 bash[66542]: cluster 2024-03-07T03:23:14.540888+0000 mon.a (mon.0) 827 : cluster 0 osdmap e603: 8 total, 8 up, 8 in 2024-03-07T03:23:15.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:15 smithi195 bash[66542]: audit 2024-03-07T03:23:15.425814+0000 mon.a (mon.0) 828 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:16.594 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:23:16.594 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:found snap1 with size 2097152 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 2 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:found snap1 with size 2097152 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:found snap2 with size 4194304 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:found snap2 with size 4194304 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 0 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:23:16.595 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3649 ms) 2024-03-07T03:23:16.596 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-03-07T03:23:16.596 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[79819]: audit 2024-03-07T03:23:15.426270+0000 mgr.y (mgr.34101) 260 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[79819]: cephadm 2024-03-07T03:23:15.428748+0000 mgr.y (mgr.34101) 261 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[79819]: cluster 2024-03-07T03:23:15.550157+0000 mon.a (mon.0) 829 : cluster 0 osdmap e604: 8 total, 8 up, 8 in 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[79819]: audit 2024-03-07T03:23:16.486170+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.148:0/2054684097' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[82222]: audit 2024-03-07T03:23:15.426270+0000 mgr.y (mgr.34101) 260 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[82222]: cephadm 2024-03-07T03:23:15.428748+0000 mgr.y (mgr.34101) 261 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[82222]: cluster 2024-03-07T03:23:15.550157+0000 mon.a (mon.0) 829 : cluster 0 osdmap e604: 8 total, 8 up, 8 in 2024-03-07T03:23:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:16 smithi148 bash[82222]: audit 2024-03-07T03:23:16.486170+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.148:0/2054684097' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:16.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:16 smithi195 bash[66542]: audit 2024-03-07T03:23:15.426270+0000 mgr.y (mgr.34101) 260 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:16.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:16 smithi195 bash[66542]: cephadm 2024-03-07T03:23:15.428748+0000 mgr.y (mgr.34101) 261 : cephadm 1 Upgrade: unsafe to stop osd(s) at this time (2 PGs are or would become offline) 2024-03-07T03:23:16.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:16 smithi195 bash[66542]: cluster 2024-03-07T03:23:15.550157+0000 mon.a (mon.0) 829 : cluster 0 osdmap e604: 8 total, 8 up, 8 in 2024-03-07T03:23:16.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:16 smithi195 bash[66542]: audit 2024-03-07T03:23:16.486170+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.148:0/2054684097' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:17.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:23:17] "GET /metrics HTTP/1.1" 200 50893 "" "Prometheus/2.43.0" 2024-03-07T03:23:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:17 smithi195 bash[66542]: cluster 2024-03-07T03:23:16.341116+0000 mgr.y (mgr.34101) 262 : cluster 0 pgmap v503: 297 pgs: 297 active+clean; 584 KiB data, 465 MiB used, 715 GiB / 715 GiB avail; 376 KiB/s rd, 23 KiB/s wr, 444 op/s 2024-03-07T03:23:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:17 smithi195 bash[66542]: cluster 2024-03-07T03:23:16.557386+0000 mon.a (mon.0) 830 : cluster 0 osdmap e605: 8 total, 8 up, 8 in 2024-03-07T03:23:17.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:17 smithi195 bash[66542]: cluster 2024-03-07T03:23:17.213992+0000 mon.a (mon.0) 831 : cluster 0 osdmap e606: 8 total, 8 up, 8 in 2024-03-07T03:23:18.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:17 smithi148 bash[79819]: cluster 2024-03-07T03:23:16.341116+0000 mgr.y (mgr.34101) 262 : cluster 0 pgmap v503: 297 pgs: 297 active+clean; 584 KiB data, 465 MiB used, 715 GiB / 715 GiB avail; 376 KiB/s rd, 23 KiB/s wr, 444 op/s 2024-03-07T03:23:18.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:17 smithi148 bash[79819]: cluster 2024-03-07T03:23:16.557386+0000 mon.a (mon.0) 830 : cluster 0 osdmap e605: 8 total, 8 up, 8 in 2024-03-07T03:23:18.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:17 smithi148 bash[79819]: cluster 2024-03-07T03:23:17.213992+0000 mon.a (mon.0) 831 : cluster 0 osdmap e606: 8 total, 8 up, 8 in 2024-03-07T03:23:18.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:17 smithi148 bash[82222]: cluster 2024-03-07T03:23:16.341116+0000 mgr.y (mgr.34101) 262 : cluster 0 pgmap v503: 297 pgs: 297 active+clean; 584 KiB data, 465 MiB used, 715 GiB / 715 GiB avail; 376 KiB/s rd, 23 KiB/s wr, 444 op/s 2024-03-07T03:23:18.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:17 smithi148 bash[82222]: cluster 2024-03-07T03:23:16.557386+0000 mon.a (mon.0) 830 : cluster 0 osdmap e605: 8 total, 8 up, 8 in 2024-03-07T03:23:18.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:17 smithi148 bash[82222]: cluster 2024-03-07T03:23:17.213992+0000 mon.a (mon.0) 831 : cluster 0 osdmap e606: 8 total, 8 up, 8 in 2024-03-07T03:23:18.262 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (1668 ms) 2024-03-07T03:23:18.262 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-03-07T03:23:18.263 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:19.256 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:23:19.256 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:23:19.256 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:23:19.256 INFO:tasks.workunit.client.0.smithi148.stdout:found snap1 with size 2097152 2024-03-07T03:23:19.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:19 smithi148 bash[79819]: cluster 2024-03-07T03:23:18.219070+0000 mon.a (mon.0) 832 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2024-03-07T03:23:19.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:19 smithi148 bash[79819]: cluster 2024-03-07T03:23:18.343834+0000 mgr.y (mgr.34101) 263 : cluster 0 pgmap v507: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 105 op/s 2024-03-07T03:23:19.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:19 smithi148 bash[79819]: audit 2024-03-07T03:23:18.840789+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.148:0/26727908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:19.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:19 smithi148 bash[82222]: cluster 2024-03-07T03:23:18.219070+0000 mon.a (mon.0) 832 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2024-03-07T03:23:19.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:19 smithi148 bash[82222]: cluster 2024-03-07T03:23:18.343834+0000 mgr.y (mgr.34101) 263 : cluster 0 pgmap v507: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 105 op/s 2024-03-07T03:23:19.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:19 smithi148 bash[82222]: audit 2024-03-07T03:23:18.840789+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.148:0/26727908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:19 smithi195 bash[66542]: cluster 2024-03-07T03:23:18.219070+0000 mon.a (mon.0) 832 : cluster 0 osdmap e607: 8 total, 8 up, 8 in 2024-03-07T03:23:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:19 smithi195 bash[66542]: cluster 2024-03-07T03:23:18.343834+0000 mgr.y (mgr.34101) 263 : cluster 0 pgmap v507: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.9 KiB/s wr, 105 op/s 2024-03-07T03:23:19.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:19 smithi195 bash[66542]: audit 2024-03-07T03:23:18.840789+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.148:0/26727908' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:20.261 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 2 2024-03-07T03:23:20.261 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:23:20.261 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:23:20.261 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:23:20.261 INFO:tasks.workunit.client.0.smithi148.stdout:found snap1 with size 2097152 2024-03-07T03:23:20.261 INFO:tasks.workunit.client.0.smithi148.stdout:found snap2 with size 4194304 2024-03-07T03:23:20.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:20 smithi148 bash[79819]: cluster 2024-03-07T03:23:19.229216+0000 mon.a (mon.0) 833 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2024-03-07T03:23:20.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:20 smithi148 bash[82222]: cluster 2024-03-07T03:23:19.229216+0000 mon.a (mon.0) 833 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2024-03-07T03:23:20.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:20 smithi195 bash[66542]: cluster 2024-03-07T03:23:19.229216+0000 mon.a (mon.0) 833 : cluster 0 osdmap e608: 8 total, 8 up, 8 in 2024-03-07T03:23:21.263 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:23:21.264 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:23:21.264 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:23:21.264 INFO:tasks.workunit.client.0.smithi148.stdout:found snap2 with size 4194304 2024-03-07T03:23:21.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:21 smithi148 bash[79819]: cluster 2024-03-07T03:23:20.229460+0000 mon.a (mon.0) 834 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2024-03-07T03:23:21.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:21 smithi148 bash[79819]: cluster 2024-03-07T03:23:20.345118+0000 mgr.y (mgr.34101) 264 : cluster 0 pgmap v510: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 6.6 KiB/s wr, 78 op/s 2024-03-07T03:23:21.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:21 smithi148 bash[79819]: audit 2024-03-07T03:23:21.198470+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.148:0/103101244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:21.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:21 smithi148 bash[82222]: cluster 2024-03-07T03:23:20.229460+0000 mon.a (mon.0) 834 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2024-03-07T03:23:21.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:21 smithi148 bash[82222]: cluster 2024-03-07T03:23:20.345118+0000 mgr.y (mgr.34101) 264 : cluster 0 pgmap v510: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 6.6 KiB/s wr, 78 op/s 2024-03-07T03:23:21.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:21 smithi148 bash[82222]: audit 2024-03-07T03:23:21.198470+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.148:0/103101244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:21 smithi195 bash[66542]: cluster 2024-03-07T03:23:20.229460+0000 mon.a (mon.0) 834 : cluster 0 osdmap e609: 8 total, 8 up, 8 in 2024-03-07T03:23:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:21 smithi195 bash[66542]: cluster 2024-03-07T03:23:20.345118+0000 mgr.y (mgr.34101) 264 : cluster 0 pgmap v510: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 6.6 KiB/s wr, 78 op/s 2024-03-07T03:23:21.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:21 smithi195 bash[66542]: audit 2024-03-07T03:23:21.198470+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.148:0/103101244' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:22.237 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 0 2024-03-07T03:23:22.237 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:23:22.253 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (3991 ms) 2024-03-07T03:23:22.253 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-03-07T03:23:22.253 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:22.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:22 smithi148 bash[79819]: cluster 2024-03-07T03:23:21.245382+0000 mon.a (mon.0) 835 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2024-03-07T03:23:22.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:22 smithi148 bash[79819]: cluster 2024-03-07T03:23:22.215166+0000 mon.a (mon.0) 836 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2024-03-07T03:23:22.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:22 smithi148 bash[82222]: cluster 2024-03-07T03:23:21.245382+0000 mon.a (mon.0) 835 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2024-03-07T03:23:22.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:22 smithi148 bash[82222]: cluster 2024-03-07T03:23:22.215166+0000 mon.a (mon.0) 836 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2024-03-07T03:23:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:22 smithi195 bash[66542]: cluster 2024-03-07T03:23:21.245382+0000 mon.a (mon.0) 835 : cluster 0 osdmap e610: 8 total, 8 up, 8 in 2024-03-07T03:23:22.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:22 smithi195 bash[66542]: cluster 2024-03-07T03:23:22.215166+0000 mon.a (mon.0) 836 : cluster 0 osdmap e611: 8 total, 8 up, 8 in 2024-03-07T03:23:22.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:23:22 smithi195 bash[59134]: ts=2024-03-07T03:23:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:23.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:23 smithi148 bash[79819]: cluster 2024-03-07T03:23:22.345920+0000 mgr.y (mgr.34101) 265 : cluster 0 pgmap v513: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:23:23.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:23 smithi148 bash[79819]: audit 2024-03-07T03:23:22.549067+0000 mon.a (mon.0) 837 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:23.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:23 smithi148 bash[79819]: cluster 2024-03-07T03:23:23.257130+0000 mon.a (mon.0) 838 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2024-03-07T03:23:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:23 smithi148 bash[82222]: cluster 2024-03-07T03:23:22.345920+0000 mgr.y (mgr.34101) 265 : cluster 0 pgmap v513: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:23:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:23 smithi148 bash[82222]: audit 2024-03-07T03:23:22.549067+0000 mon.a (mon.0) 837 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:23 smithi148 bash[82222]: cluster 2024-03-07T03:23:23.257130+0000 mon.a (mon.0) 838 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2024-03-07T03:23:23.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:23 smithi195 bash[66542]: cluster 2024-03-07T03:23:22.345920+0000 mgr.y (mgr.34101) 265 : cluster 0 pgmap v513: 297 pgs: 297 active+clean; 584 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:23:23.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:23 smithi195 bash[66542]: audit 2024-03-07T03:23:22.549067+0000 mon.a (mon.0) 837 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:23.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:23 smithi195 bash[66542]: cluster 2024-03-07T03:23:23.257130+0000 mon.a (mon.0) 838 : cluster 0 osdmap e612: 8 total, 8 up, 8 in 2024-03-07T03:23:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:24 smithi148 bash[79819]: audit 2024-03-07T03:23:23.567918+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.148:0/3633276759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:24 smithi148 bash[79819]: cluster 2024-03-07T03:23:24.263875+0000 mon.a (mon.0) 839 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2024-03-07T03:23:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:24 smithi148 bash[82222]: audit 2024-03-07T03:23:23.567918+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.148:0/3633276759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:24.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:24 smithi148 bash[82222]: cluster 2024-03-07T03:23:24.263875+0000 mon.a (mon.0) 839 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2024-03-07T03:23:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:24 smithi195 bash[66542]: audit 2024-03-07T03:23:23.567918+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.148:0/3633276759' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:24 smithi195 bash[66542]: cluster 2024-03-07T03:23:24.263875+0000 mon.a (mon.0) 839 : cluster 0 osdmap e613: 8 total, 8 up, 8 in 2024-03-07T03:23:25.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:25 smithi148 bash[79819]: cluster 2024-03-07T03:23:24.346775+0000 mgr.y (mgr.34101) 266 : cluster 0 pgmap v516: 297 pgs: 297 active+clean; 584 KiB data, 454 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 6.5 KiB/s wr, 81 op/s 2024-03-07T03:23:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:25 smithi148 bash[79819]: cluster 2024-03-07T03:23:25.273045+0000 mon.a (mon.0) 840 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2024-03-07T03:23:25.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:25 smithi148 bash[82222]: cluster 2024-03-07T03:23:24.346775+0000 mgr.y (mgr.34101) 266 : cluster 0 pgmap v516: 297 pgs: 297 active+clean; 584 KiB data, 454 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 6.5 KiB/s wr, 81 op/s 2024-03-07T03:23:25.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:25 smithi148 bash[82222]: cluster 2024-03-07T03:23:25.273045+0000 mon.a (mon.0) 840 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2024-03-07T03:23:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:25 smithi195 bash[66542]: cluster 2024-03-07T03:23:24.346775+0000 mgr.y (mgr.34101) 266 : cluster 0 pgmap v516: 297 pgs: 297 active+clean; 584 KiB data, 454 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 6.5 KiB/s wr, 81 op/s 2024-03-07T03:23:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:25 smithi195 bash[66542]: cluster 2024-03-07T03:23:25.273045+0000 mon.a (mon.0) 840 : cluster 0 osdmap e614: 8 total, 8 up, 8 in 2024-03-07T03:23:26.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:26 smithi148 bash[79819]: audit 2024-03-07T03:23:25.923226+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.148:0/1670273601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:26.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:26 smithi148 bash[79819]: cluster 2024-03-07T03:23:26.274986+0000 mon.a (mon.0) 841 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2024-03-07T03:23:26.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:26 smithi148 bash[82222]: audit 2024-03-07T03:23:25.923226+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.148:0/1670273601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:26.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:26 smithi148 bash[82222]: cluster 2024-03-07T03:23:26.274986+0000 mon.a (mon.0) 841 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2024-03-07T03:23:26.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:26 smithi195 bash[66542]: audit 2024-03-07T03:23:25.923226+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.148:0/1670273601' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:26.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:26 smithi195 bash[66542]: cluster 2024-03-07T03:23:26.274986+0000 mon.a (mon.0) 841 : cluster 0 osdmap e615: 8 total, 8 up, 8 in 2024-03-07T03:23:27.538 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:23:27] "GET /metrics HTTP/1.1" 200 50893 "" "Prometheus/2.43.0" 2024-03-07T03:23:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:27 smithi195 bash[66542]: cluster 2024-03-07T03:23:26.347843+0000 mgr.y (mgr.34101) 267 : cluster 0 pgmap v519: 297 pgs: 297 active+clean; 584 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 6.5 KiB/s wr, 81 op/s 2024-03-07T03:23:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:27 smithi195 bash[66542]: cluster 2024-03-07T03:23:27.218219+0000 mon.a (mon.0) 842 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2024-03-07T03:23:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:27 smithi195 bash[66542]: audit 2024-03-07T03:23:27.314257+0000 mon.a (mon.0) 843 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:27 smithi148 bash[79819]: cluster 2024-03-07T03:23:26.347843+0000 mgr.y (mgr.34101) 267 : cluster 0 pgmap v519: 297 pgs: 297 active+clean; 584 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 6.5 KiB/s wr, 81 op/s 2024-03-07T03:23:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:27 smithi148 bash[79819]: cluster 2024-03-07T03:23:27.218219+0000 mon.a (mon.0) 842 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2024-03-07T03:23:28.038 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:27 smithi148 bash[79819]: audit 2024-03-07T03:23:27.314257+0000 mon.a (mon.0) 843 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:27 smithi148 bash[82222]: cluster 2024-03-07T03:23:26.347843+0000 mgr.y (mgr.34101) 267 : cluster 0 pgmap v519: 297 pgs: 297 active+clean; 584 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s rd, 6.5 KiB/s wr, 81 op/s 2024-03-07T03:23:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:27 smithi148 bash[82222]: cluster 2024-03-07T03:23:27.218219+0000 mon.a (mon.0) 842 : cluster 0 osdmap e616: 8 total, 8 up, 8 in 2024-03-07T03:23:28.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:27 smithi148 bash[82222]: audit 2024-03-07T03:23:27.314257+0000 mon.a (mon.0) 843 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:28.283 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 0 2024-03-07T03:23:28.283 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:23:28.294 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (6041 ms) 2024-03-07T03:23:28.294 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-03-07T03:23:28.294 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:29.311 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:23:29.311 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:23:29.311 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:23:29.311 INFO:tasks.workunit.client.0.smithi148.stdout:found snap1 with size 2097152 2024-03-07T03:23:29.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:29 smithi148 bash[79819]: cluster 2024-03-07T03:23:28.281613+0000 mon.a (mon.0) 844 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2024-03-07T03:23:29.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:29 smithi148 bash[79819]: audit 2024-03-07T03:23:28.288700+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.148:0/1462708932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:29.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:29 smithi148 bash[79819]: cluster 2024-03-07T03:23:28.348780+0000 mgr.y (mgr.34101) 268 : cluster 0 pgmap v522: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 4.7 KiB/s wr, 58 op/s 2024-03-07T03:23:29.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:29 smithi148 bash[82222]: cluster 2024-03-07T03:23:28.281613+0000 mon.a (mon.0) 844 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2024-03-07T03:23:29.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:29 smithi148 bash[82222]: audit 2024-03-07T03:23:28.288700+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.148:0/1462708932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:29.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:29 smithi148 bash[82222]: cluster 2024-03-07T03:23:28.348780+0000 mgr.y (mgr.34101) 268 : cluster 0 pgmap v522: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 4.7 KiB/s wr, 58 op/s 2024-03-07T03:23:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:29 smithi195 bash[66542]: cluster 2024-03-07T03:23:28.281613+0000 mon.a (mon.0) 844 : cluster 0 osdmap e617: 8 total, 8 up, 8 in 2024-03-07T03:23:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:29 smithi195 bash[66542]: audit 2024-03-07T03:23:28.288700+0000 mon.a (mon.0) 845 : audit 0 from='client.? 172.21.15.148:0/1462708932' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:29.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:29 smithi195 bash[66542]: cluster 2024-03-07T03:23:28.348780+0000 mgr.y (mgr.34101) 268 : cluster 0 pgmap v522: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 4.7 KiB/s wr, 58 op/s 2024-03-07T03:23:30.326 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 2 2024-03-07T03:23:30.326 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:23:30.326 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1 2024-03-07T03:23:30.326 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:23:30.326 INFO:tasks.workunit.client.0.smithi148.stdout:found snap1 with size 2097152 2024-03-07T03:23:30.326 INFO:tasks.workunit.client.0.smithi148.stdout:found snap2 with size 4194304 2024-03-07T03:23:30.332 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 2 2024-03-07T03:23:30.332 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:23:30.332 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap1-rename 2024-03-07T03:23:30.332 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2 2024-03-07T03:23:30.332 INFO:tasks.workunit.client.0.smithi148.stdout:found snap1-rename with size 2097152 2024-03-07T03:23:30.332 INFO:tasks.workunit.client.0.smithi148.stdout:found snap2 with size 4194304 2024-03-07T03:23:30.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:30 smithi148 bash[82222]: cluster 2024-03-07T03:23:29.283463+0000 mon.a (mon.0) 846 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2024-03-07T03:23:30.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:30 smithi148 bash[79819]: cluster 2024-03-07T03:23:29.283463+0000 mon.a (mon.0) 846 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2024-03-07T03:23:30.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:30 smithi195 bash[66542]: cluster 2024-03-07T03:23:29.283463+0000 mon.a (mon.0) 846 : cluster 0 osdmap e618: 8 total, 8 up, 8 in 2024-03-07T03:23:31.341 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:23:31.341 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:23:31.341 INFO:tasks.workunit.client.0.smithi148.stdout:snap: snap2-rename 2024-03-07T03:23:31.341 INFO:tasks.workunit.client.0.smithi148.stdout:found snap2-rename with size 4194304 2024-03-07T03:23:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:31 smithi195 bash[66542]: cluster 2024-03-07T03:23:30.297138+0000 mon.a (mon.0) 847 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2024-03-07T03:23:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:31 smithi195 bash[66542]: cluster 2024-03-07T03:23:30.349346+0000 mgr.y (mgr.34101) 269 : cluster 0 pgmap v525: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 4.5 KiB/s wr, 54 op/s 2024-03-07T03:23:31.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:31 smithi195 bash[66542]: audit 2024-03-07T03:23:30.440488+0000 mon.a (mon.0) 848 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:31 smithi195 bash[66542]: audit 2024-03-07T03:23:30.440663+0000 mgr.y (mgr.34101) 270 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:31 smithi195 bash[66542]: cephadm 2024-03-07T03:23:30.441939+0000 mgr.y (mgr.34101) 271 : cephadm 1 Upgrade: osd.2 is safe to restart 2024-03-07T03:23:31.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:31 smithi195 bash[66542]: audit 2024-03-07T03:23:30.653098+0000 mon.a (mon.0) 849 : audit 0 from='client.? 172.21.15.148:0/3271864212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[79819]: cluster 2024-03-07T03:23:30.297138+0000 mon.a (mon.0) 847 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2024-03-07T03:23:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[79819]: cluster 2024-03-07T03:23:30.349346+0000 mgr.y (mgr.34101) 269 : cluster 0 pgmap v525: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 4.5 KiB/s wr, 54 op/s 2024-03-07T03:23:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[79819]: audit 2024-03-07T03:23:30.440488+0000 mon.a (mon.0) 848 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[79819]: audit 2024-03-07T03:23:30.440663+0000 mgr.y (mgr.34101) 270 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[79819]: cephadm 2024-03-07T03:23:30.441939+0000 mgr.y (mgr.34101) 271 : cephadm 1 Upgrade: osd.2 is safe to restart 2024-03-07T03:23:31.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[79819]: audit 2024-03-07T03:23:30.653098+0000 mon.a (mon.0) 849 : audit 0 from='client.? 172.21.15.148:0/3271864212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:31.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[82222]: cluster 2024-03-07T03:23:30.297138+0000 mon.a (mon.0) 847 : cluster 0 osdmap e619: 8 total, 8 up, 8 in 2024-03-07T03:23:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[82222]: cluster 2024-03-07T03:23:30.349346+0000 mgr.y (mgr.34101) 269 : cluster 0 pgmap v525: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 4.5 KiB/s wr, 54 op/s 2024-03-07T03:23:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[82222]: audit 2024-03-07T03:23:30.440488+0000 mon.a (mon.0) 848 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[82222]: audit 2024-03-07T03:23:30.440663+0000 mgr.y (mgr.34101) 270 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-07T03:23:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[82222]: cephadm 2024-03-07T03:23:30.441939+0000 mgr.y (mgr.34101) 271 : cephadm 1 Upgrade: osd.2 is safe to restart 2024-03-07T03:23:31.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:31 smithi148 bash[82222]: audit 2024-03-07T03:23:30.653098+0000 mon.a (mon.0) 849 : audit 0 from='client.? 172.21.15.148:0/3271864212' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:32.216 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 0 2024-03-07T03:23:32.216 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:23:32.231 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (3938 ms) 2024-03-07T03:23:32.232 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-03-07T03:23:32.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:23:32 smithi195 bash[59134]: ts=2024-03-07T03:23:32.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:32.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:32 smithi148 bash[79819]: cluster 2024-03-07T03:23:31.316780+0000 mon.a (mon.0) 850 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2024-03-07T03:23:32.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:32 smithi148 bash[79819]: cluster 2024-03-07T03:23:32.216613+0000 mon.a (mon.0) 851 : cluster 0 osdmap e621: 8 total, 8 up, 8 in 2024-03-07T03:23:32.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:32 smithi148 bash[82222]: cluster 2024-03-07T03:23:31.316780+0000 mon.a (mon.0) 850 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2024-03-07T03:23:32.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:32 smithi148 bash[82222]: cluster 2024-03-07T03:23:32.216613+0000 mon.a (mon.0) 851 : cluster 0 osdmap e621: 8 total, 8 up, 8 in 2024-03-07T03:23:33.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:32 smithi195 bash[66542]: cluster 2024-03-07T03:23:31.316780+0000 mon.a (mon.0) 850 : cluster 0 osdmap e620: 8 total, 8 up, 8 in 2024-03-07T03:23:33.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:32 smithi195 bash[66542]: cluster 2024-03-07T03:23:32.216613+0000 mon.a (mon.0) 851 : cluster 0 osdmap e621: 8 total, 8 up, 8 in 2024-03-07T03:23:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: cluster 2024-03-07T03:23:32.349818+0000 mgr.y (mgr.34101) 272 : cluster 0 pgmap v528: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 2.5 KiB/s wr, 41 op/s 2024-03-07T03:23:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: cephadm 2024-03-07T03:23:32.960969+0000 mgr.y (mgr.34101) 273 : cephadm 1 Upgrade: Updating osd.2 2024-03-07T03:23:34.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: audit 2024-03-07T03:23:32.965559+0000 mon.a (mon.0) 852 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: audit 2024-03-07T03:23:32.967881+0000 mon.a (mon.0) 853 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:23:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: audit 2024-03-07T03:23:32.968658+0000 mon.a (mon.0) 854 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: cephadm 2024-03-07T03:23:32.977227+0000 mgr.y (mgr.34101) 274 : cephadm 1 Deploying daemon osd.2 on smithi148 2024-03-07T03:23:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: audit 2024-03-07T03:23:33.004234+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.148:0/3953755015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: cluster 2024-03-07T03:23:33.227835+0000 mon.a (mon.0) 855 : cluster 0 osdmap e622: 8 total, 8 up, 8 in 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[79819]: audit 2024-03-07T03:23:33.229482+0000 mon.a (mon.0) 856 : audit 1 from='client.? 172.21.15.148:0/3898303905' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: cluster 2024-03-07T03:23:32.349818+0000 mgr.y (mgr.34101) 272 : cluster 0 pgmap v528: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 2.5 KiB/s wr, 41 op/s 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: cephadm 2024-03-07T03:23:32.960969+0000 mgr.y (mgr.34101) 273 : cephadm 1 Upgrade: Updating osd.2 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: audit 2024-03-07T03:23:32.965559+0000 mon.a (mon.0) 852 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: audit 2024-03-07T03:23:32.967881+0000 mon.a (mon.0) 853 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: audit 2024-03-07T03:23:32.968658+0000 mon.a (mon.0) 854 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: cephadm 2024-03-07T03:23:32.977227+0000 mgr.y (mgr.34101) 274 : cephadm 1 Deploying daemon osd.2 on smithi148 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: audit 2024-03-07T03:23:33.004234+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.148:0/3953755015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: cluster 2024-03-07T03:23:33.227835+0000 mon.a (mon.0) 855 : cluster 0 osdmap e622: 8 total, 8 up, 8 in 2024-03-07T03:23:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:33 smithi148 bash[82222]: audit 2024-03-07T03:23:33.229482+0000 mon.a (mon.0) 856 : audit 1 from='client.? 172.21.15.148:0/3898303905' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:34.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: cluster 2024-03-07T03:23:32.349818+0000 mgr.y (mgr.34101) 272 : cluster 0 pgmap v528: 297 pgs: 297 active+clean; 584 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 2.5 KiB/s wr, 41 op/s 2024-03-07T03:23:34.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: cephadm 2024-03-07T03:23:32.960969+0000 mgr.y (mgr.34101) 273 : cephadm 1 Upgrade: Updating osd.2 2024-03-07T03:23:34.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: audit 2024-03-07T03:23:32.965559+0000 mon.a (mon.0) 852 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:34.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: audit 2024-03-07T03:23:32.967881+0000 mon.a (mon.0) 853 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-07T03:23:34.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: audit 2024-03-07T03:23:32.968658+0000 mon.a (mon.0) 854 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:34.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: cephadm 2024-03-07T03:23:32.977227+0000 mgr.y (mgr.34101) 274 : cephadm 1 Deploying daemon osd.2 on smithi148 2024-03-07T03:23:34.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: audit 2024-03-07T03:23:33.004234+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.148:0/3953755015' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:34.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: cluster 2024-03-07T03:23:33.227835+0000 mon.a (mon.0) 855 : cluster 0 osdmap e622: 8 total, 8 up, 8 in 2024-03-07T03:23:34.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:33 smithi195 bash[66542]: audit 2024-03-07T03:23:33.229482+0000 mon.a (mon.0) 856 : audit 1 from='client.? 172.21.15.148:0/3898303905' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-22","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:34.243 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:34.243 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:34.243 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:34.247 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:35.538 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:35 smithi148 bash[82222]: audit 2024-03-07T03:23:34.225675+0000 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.148:0/3898303905' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:35.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:35 smithi148 bash[82222]: cluster 2024-03-07T03:23:34.233136+0000 mon.a (mon.0) 858 : cluster 0 osdmap e623: 8 total, 8 up, 8 in 2024-03-07T03:23:35.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:35 smithi148 bash[82222]: cluster 2024-03-07T03:23:34.350549+0000 mgr.y (mgr.34101) 275 : cluster 0 pgmap v531: 329 pgs: 23 creating+peering, 9 unknown, 297 active+clean; 584 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 KiB/s wr, 61 op/s 2024-03-07T03:23:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:35 smithi148 bash[79819]: audit 2024-03-07T03:23:34.225675+0000 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.148:0/3898303905' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:35 smithi148 bash[79819]: cluster 2024-03-07T03:23:34.233136+0000 mon.a (mon.0) 858 : cluster 0 osdmap e623: 8 total, 8 up, 8 in 2024-03-07T03:23:35.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:35 smithi148 bash[79819]: cluster 2024-03-07T03:23:34.350549+0000 mgr.y (mgr.34101) 275 : cluster 0 pgmap v531: 329 pgs: 23 creating+peering, 9 unknown, 297 active+clean; 584 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 KiB/s wr, 61 op/s 2024-03-07T03:23:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:35 smithi195 bash[66542]: audit 2024-03-07T03:23:34.225675+0000 mon.a (mon.0) 857 : audit 1 from='client.? 172.21.15.148:0/3898303905' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:35 smithi195 bash[66542]: cluster 2024-03-07T03:23:34.233136+0000 mon.a (mon.0) 858 : cluster 0 osdmap e623: 8 total, 8 up, 8 in 2024-03-07T03:23:35.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:35 smithi195 bash[66542]: cluster 2024-03-07T03:23:34.350549+0000 mgr.y (mgr.34101) 275 : cluster 0 pgmap v531: 329 pgs: 23 creating+peering, 9 unknown, 297 active+clean; 584 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.2 KiB/s wr, 61 op/s 2024-03-07T03:23:36.529 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:36 smithi148 bash[79819]: cluster 2024-03-07T03:23:35.238124+0000 mon.a (mon.0) 859 : cluster 0 osdmap e624: 8 total, 8 up, 8 in 2024-03-07T03:23:36.530 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:36 smithi148 bash[79819]: audit 2024-03-07T03:23:35.369114+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.148:0/718340397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:36.530 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:36 smithi148 bash[82222]: cluster 2024-03-07T03:23:35.238124+0000 mon.a (mon.0) 859 : cluster 0 osdmap e624: 8 total, 8 up, 8 in 2024-03-07T03:23:36.530 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:36 smithi148 bash[82222]: audit 2024-03-07T03:23:35.369114+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.148:0/718340397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:36.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:36 smithi195 bash[66542]: cluster 2024-03-07T03:23:35.238124+0000 mon.a (mon.0) 859 : cluster 0 osdmap e624: 8 total, 8 up, 8 in 2024-03-07T03:23:36.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:36 smithi195 bash[66542]: audit 2024-03-07T03:23:35.369114+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.148:0/718340397' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:36.659 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4425 ms) 2024-03-07T03:23:36.659 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-03-07T03:23:37.435 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:23:37] "GET /metrics HTTP/1.1" 200 50943 "" "Prometheus/2.43.0" 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[79819]: cluster 2024-03-07T03:23:36.250092+0000 mon.a (mon.0) 860 : cluster 0 osdmap e625: 8 total, 8 up, 8 in 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[79819]: cluster 2024-03-07T03:23:36.351078+0000 mgr.y (mgr.34101) 276 : cluster 0 pgmap v534: 329 pgs: 23 creating+peering, 306 active+clean; 584 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 42 op/s 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[79819]: cluster 2024-03-07T03:23:37.224323+0000 mon.a (mon.0) 861 : cluster 0 osdmap e626: 8 total, 8 up, 8 in 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[79819]: audit 2024-03-07T03:23:37.227562+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.148:0/1498713789' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[79819]: audit 2024-03-07T03:23:37.228407+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[82222]: cluster 2024-03-07T03:23:36.250092+0000 mon.a (mon.0) 860 : cluster 0 osdmap e625: 8 total, 8 up, 8 in 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[82222]: cluster 2024-03-07T03:23:36.351078+0000 mgr.y (mgr.34101) 276 : cluster 0 pgmap v534: 329 pgs: 23 creating+peering, 306 active+clean; 584 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 42 op/s 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[82222]: cluster 2024-03-07T03:23:37.224323+0000 mon.a (mon.0) 861 : cluster 0 osdmap e626: 8 total, 8 up, 8 in 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[82222]: audit 2024-03-07T03:23:37.227562+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.148:0/1498713789' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:37.436 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[82222]: audit 2024-03-07T03:23:37.228407+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:37 smithi195 bash[66542]: cluster 2024-03-07T03:23:36.250092+0000 mon.a (mon.0) 860 : cluster 0 osdmap e625: 8 total, 8 up, 8 in 2024-03-07T03:23:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:37 smithi195 bash[66542]: cluster 2024-03-07T03:23:36.351078+0000 mgr.y (mgr.34101) 276 : cluster 0 pgmap v534: 329 pgs: 23 creating+peering, 306 active+clean; 584 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 42 op/s 2024-03-07T03:23:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:37 smithi195 bash[66542]: cluster 2024-03-07T03:23:37.224323+0000 mon.a (mon.0) 861 : cluster 0 osdmap e626: 8 total, 8 up, 8 in 2024-03-07T03:23:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:37 smithi195 bash[66542]: audit 2024-03-07T03:23:37.227562+0000 mon.c (mon.2) 38 : audit 1 from='client.? 172.21.15.148:0/1498713789' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:37.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:37 smithi195 bash[66542]: audit 2024-03-07T03:23:37.228407+0000 mon.a (mon.0) 862 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:37.691 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.692 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.692 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.692 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.693 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.693 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.693 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.694 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:37.694 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:38.039 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:37 smithi148 systemd[1]: Stopping Ceph osd.2 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:23:38.039 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[45095]: debug 2024-03-07T03:23:37.798+0000 7f02f4eef700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:23:38.039 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[45095]: debug 2024-03-07T03:23:37.798+0000 7f02f4eef700 -1 osd.2 626 *** Got signal Terminated *** 2024-03-07T03:23:38.039 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:37 smithi148 bash[45095]: debug 2024-03-07T03:23:37.798+0000 7f02f4eef700 -1 osd.2 626 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:23:38.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[82222]: audit 2024-03-07T03:23:37.719642+0000 mon.a (mon.0) 863 : audit 0 from='client.? 172.21.15.148:0/3847124728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:38.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[82222]: cluster 2024-03-07T03:23:37.803261+0000 mon.a (mon.0) 864 : cluster 1 osd.2 marked itself down and dead 2024-03-07T03:23:38.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[82222]: cluster 2024-03-07T03:23:38.220023+0000 mon.a (mon.0) 865 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:23:38.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[82222]: cluster 2024-03-07T03:23:38.220122+0000 mon.a (mon.0) 866 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:38.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[82222]: audit 2024-03-07T03:23:38.222512+0000 mon.a (mon.0) 867 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:38.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[82222]: cluster 2024-03-07T03:23:38.232827+0000 mon.a (mon.0) 868 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2024-03-07T03:23:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[79819]: debug 2024-03-07T03:23:38.214+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[79819]: audit 2024-03-07T03:23:37.719642+0000 mon.a (mon.0) 863 : audit 0 from='client.? 172.21.15.148:0/3847124728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[79819]: cluster 2024-03-07T03:23:37.803261+0000 mon.a (mon.0) 864 : cluster 1 osd.2 marked itself down and dead 2024-03-07T03:23:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[79819]: cluster 2024-03-07T03:23:38.220023+0000 mon.a (mon.0) 865 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:23:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[79819]: cluster 2024-03-07T03:23:38.220122+0000 mon.a (mon.0) 866 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[79819]: audit 2024-03-07T03:23:38.222512+0000 mon.a (mon.0) 867 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:38 smithi148 bash[79819]: cluster 2024-03-07T03:23:38.232827+0000 mon.a (mon.0) 868 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2024-03-07T03:23:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:38 smithi195 bash[66542]: audit 2024-03-07T03:23:37.719642+0000 mon.a (mon.0) 863 : audit 0 from='client.? 172.21.15.148:0/3847124728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:38 smithi195 bash[66542]: cluster 2024-03-07T03:23:37.803261+0000 mon.a (mon.0) 864 : cluster 1 osd.2 marked itself down and dead 2024-03-07T03:23:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:38 smithi195 bash[66542]: cluster 2024-03-07T03:23:38.220023+0000 mon.a (mon.0) 865 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:23:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:38 smithi195 bash[66542]: cluster 2024-03-07T03:23:38.220122+0000 mon.a (mon.0) 866 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:38.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:38 smithi195 bash[66542]: audit 2024-03-07T03:23:38.222512+0000 mon.a (mon.0) 867 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:23:38.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:38 smithi195 bash[66542]: cluster 2024-03-07T03:23:38.232827+0000 mon.a (mon.0) 868 : cluster 0 osdmap e627: 8 total, 7 up, 8 in 2024-03-07T03:23:39.306 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:39.538 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[99138]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:23:39.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[79819]: cluster 2024-03-07T03:23:38.353315+0000 mgr.y (mgr.34101) 277 : cluster 0 pgmap v537: 361 pgs: 245 active+clean, 9 creating+activating, 2 stale+creating+peering, 13 stale+active+clean, 74 peering, 9 unknown, 9 creating+peering; 584 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 8.0 KiB/s wr, 130 op/s 2024-03-07T03:23:39.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[79819]: cluster 2024-03-07T03:23:39.236485+0000 mon.a (mon.0) 869 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2024-03-07T03:23:39.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[72584]: [07/Mar/2024:03:23:39] ENGINE Bus STOPPING 2024-03-07T03:23:39.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[72584]: [07/Mar/2024:03:23:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:23:39.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[72584]: [07/Mar/2024:03:23:39] ENGINE Bus STOPPED 2024-03-07T03:23:39.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[72584]: [07/Mar/2024:03:23:39] ENGINE Bus STARTING 2024-03-07T03:23:39.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[72584]: [07/Mar/2024:03:23:39] ENGINE Serving on http://:::9283 2024-03-07T03:23:39.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[72584]: [07/Mar/2024:03:23:39] ENGINE Bus STARTED 2024-03-07T03:23:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[82222]: cluster 2024-03-07T03:23:38.353315+0000 mgr.y (mgr.34101) 277 : cluster 0 pgmap v537: 361 pgs: 245 active+clean, 9 creating+activating, 2 stale+creating+peering, 13 stale+active+clean, 74 peering, 9 unknown, 9 creating+peering; 584 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 8.0 KiB/s wr, 130 op/s 2024-03-07T03:23:39.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[82222]: cluster 2024-03-07T03:23:39.236485+0000 mon.a (mon.0) 869 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2024-03-07T03:23:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:39 smithi195 bash[66542]: cluster 2024-03-07T03:23:38.353315+0000 mgr.y (mgr.34101) 277 : cluster 0 pgmap v537: 361 pgs: 245 active+clean, 9 creating+activating, 2 stale+creating+peering, 13 stale+active+clean, 74 peering, 9 unknown, 9 creating+peering; 584 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 8.0 KiB/s wr, 130 op/s 2024-03-07T03:23:39.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:39 smithi195 bash[66542]: cluster 2024-03-07T03:23:39.236485+0000 mon.a (mon.0) 869 : cluster 0 osdmap e628: 8 total, 7 up, 8 in 2024-03-07T03:23:40.038 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:39 smithi148 bash[99216]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:23:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[79819]: cluster 2024-03-07T03:23:39.266743+0000 mon.a (mon.0) 870 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:23:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[79819]: audit 2024-03-07T03:23:39.320352+0000 mon.a (mon.0) 871 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:23:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[79819]: audit 2024-03-07T03:23:40.065078+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.148:0/1210256501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:40.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[79819]: cluster 2024-03-07T03:23:40.281589+0000 mon.a (mon.0) 872 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2024-03-07T03:23:40.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[82222]: cluster 2024-03-07T03:23:39.266743+0000 mon.a (mon.0) 870 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:23:40.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[82222]: audit 2024-03-07T03:23:39.320352+0000 mon.a (mon.0) 871 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:23:40.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[82222]: audit 2024-03-07T03:23:40.065078+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.148:0/1210256501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:40 smithi148 bash[82222]: cluster 2024-03-07T03:23:40.281589+0000 mon.a (mon.0) 872 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2024-03-07T03:23:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:40 smithi195 bash[66542]: cluster 2024-03-07T03:23:39.266743+0000 mon.a (mon.0) 870 : cluster 3 Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-03-07T03:23:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:40 smithi195 bash[66542]: audit 2024-03-07T03:23:39.320352+0000 mon.a (mon.0) 871 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:23:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:40 smithi195 bash[66542]: audit 2024-03-07T03:23:40.065078+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.148:0/1210256501' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:40.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:40 smithi195 bash[66542]: cluster 2024-03-07T03:23:40.281589+0000 mon.a (mon.0) 872 : cluster 0 osdmap e629: 8 total, 7 up, 8 in 2024-03-07T03:23:41.484 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:23:41.660 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:41 smithi148 bash[82222]: cluster 2024-03-07T03:23:40.353955+0000 mgr.y (mgr.34101) 278 : cluster 0 pgmap v540: 361 pgs: 13 active+undersized, 7 active+undersized+degraded, 237 active+clean, 9 creating+activating, 2 stale+creating+peering, 7 stale+active+clean, 74 peering, 3 unknown, 9 creating+peering; 584 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 6.5 KiB/s wr, 105 op/s; 11/486 objects degraded (2.263%) 2024-03-07T03:23:41.660 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:41 smithi148 bash[82222]: cluster 2024-03-07T03:23:41.304583+0000 mon.a (mon.0) 873 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2024-03-07T03:23:41.660 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:41 smithi148 bash[79819]: cluster 2024-03-07T03:23:40.353955+0000 mgr.y (mgr.34101) 278 : cluster 0 pgmap v540: 361 pgs: 13 active+undersized, 7 active+undersized+degraded, 237 active+clean, 9 creating+activating, 2 stale+creating+peering, 7 stale+active+clean, 74 peering, 3 unknown, 9 creating+peering; 584 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 6.5 KiB/s wr, 105 op/s; 11/486 objects degraded (2.263%) 2024-03-07T03:23:41.660 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:41 smithi148 bash[79819]: cluster 2024-03-07T03:23:41.304583+0000 mon.a (mon.0) 873 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2024-03-07T03:23:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:41 smithi195 bash[66542]: cluster 2024-03-07T03:23:40.353955+0000 mgr.y (mgr.34101) 278 : cluster 0 pgmap v540: 361 pgs: 13 active+undersized, 7 active+undersized+degraded, 237 active+clean, 9 creating+activating, 2 stale+creating+peering, 7 stale+active+clean, 74 peering, 3 unknown, 9 creating+peering; 584 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s rd, 6.5 KiB/s wr, 105 op/s; 11/486 objects degraded (2.263%) 2024-03-07T03:23:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:41 smithi195 bash[66542]: cluster 2024-03-07T03:23:41.304583+0000 mon.a (mon.0) 873 : cluster 0 osdmap e630: 8 total, 7 up, 8 in 2024-03-07T03:23:41.935 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.936 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.936 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.936 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.937 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.937 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.937 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.938 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.938 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-07T03:23:41.938 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: Stopped Ceph osd.2 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:23:41.938 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:23:41.938 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:41 smithi148 systemd[1]: Started Ceph osd.2 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:23:42.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[59134]: ts=2024-03-07T03:23:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: cluster 2024-03-07T03:23:41.308450+0000 mon.a (mon.0) 874 : cluster 3 Health check failed: Degraded data redundancy: 11/486 objects degraded (2.263%), 7 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:41.520114+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:41.520630+0000 mon.a (mon.0) 875 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:41.992198+0000 mon.a (mon.0) 876 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:41.997974+0000 mon.a (mon.0) 877 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:42.016921+0000 mon.a (mon.0) 878 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:42.218728+0000 mon.a (mon.0) 879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: cluster 2024-03-07T03:23:42.224381+0000 mon.a (mon.0) 880 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:42.316994+0000 mon.a (mon.0) 881 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:42 smithi195 bash[66542]: audit 2024-03-07T03:23:42.318241+0000 mon.a (mon.0) 882 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:42.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: cluster 2024-03-07T03:23:41.308450+0000 mon.a (mon.0) 874 : cluster 3 Health check failed: Degraded data redundancy: 11/486 objects degraded (2.263%), 7 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:42.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:41.520114+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:23:42.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:41.520630+0000 mon.a (mon.0) 875 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:23:42.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:41.992198+0000 mon.a (mon.0) 876 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:41.997974+0000 mon.a (mon.0) 877 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:42.016921+0000 mon.a (mon.0) 878 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:42.218728+0000 mon.a (mon.0) 879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: cluster 2024-03-07T03:23:42.224381+0000 mon.a (mon.0) 880 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:42.316994+0000 mon.a (mon.0) 881 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[79819]: audit 2024-03-07T03:23:42.318241+0000 mon.a (mon.0) 882 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: cluster 2024-03-07T03:23:41.308450+0000 mon.a (mon.0) 874 : cluster 3 Health check failed: Degraded data redundancy: 11/486 objects degraded (2.263%), 7 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:41.520114+0000 mon.c (mon.2) 40 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:41.520630+0000 mon.a (mon.0) 875 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:23:42.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:41.992198+0000 mon.a (mon.0) 876 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:41.997974+0000 mon.a (mon.0) 877 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:42.016921+0000 mon.a (mon.0) 878 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:42.218728+0000 mon.a (mon.0) 879 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:23:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: cluster 2024-03-07T03:23:42.224381+0000 mon.a (mon.0) 880 : cluster 0 osdmap e631: 8 total, 7 up, 8 in 2024-03-07T03:23:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:42.316994+0000 mon.a (mon.0) 881 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:42 smithi148 bash[82222]: audit 2024-03-07T03:23:42.318241+0000 mon.a (mon.0) 882 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:43.658 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[79819]: cluster 2024-03-07T03:23:42.354638+0000 mgr.y (mgr.34101) 279 : cluster 0 pgmap v543: 361 pgs: 30 active+undersized, 13 active+undersized+degraded, 222 active+clean, 9 creating+activating, 1 stale+creating+peering, 3 stale+active+clean, 74 peering, 9 creating+peering; 584 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 31 op/s; 21/486 objects degraded (4.321%) 2024-03-07T03:23:43.658 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[79819]: audit 2024-03-07T03:23:42.434465+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.148:0/3648191181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:43.658 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[82222]: cluster 2024-03-07T03:23:42.354638+0000 mgr.y (mgr.34101) 279 : cluster 0 pgmap v543: 361 pgs: 30 active+undersized, 13 active+undersized+degraded, 222 active+clean, 9 creating+activating, 1 stale+creating+peering, 3 stale+active+clean, 74 peering, 9 creating+peering; 584 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 31 op/s; 21/486 objects degraded (4.321%) 2024-03-07T03:23:43.659 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[82222]: audit 2024-03-07T03:23:42.434465+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.148:0/3648191181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:43.659 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[99391]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-07T03:23:43.659 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[99391]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-07T03:23:43.659 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[99391]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-07T03:23:43.659 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[99391]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-07T03:23:43.659 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[99391]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-07T03:23:43.659 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[99391]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-07T03:23:43.659 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:43 smithi148 bash[99391]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-07T03:23:43.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:43 smithi195 bash[66542]: cluster 2024-03-07T03:23:42.354638+0000 mgr.y (mgr.34101) 279 : cluster 0 pgmap v543: 361 pgs: 30 active+undersized, 13 active+undersized+degraded, 222 active+clean, 9 creating+activating, 1 stale+creating+peering, 3 stale+active+clean, 74 peering, 9 creating+peering; 584 KiB data, 428 MiB used, 715 GiB / 715 GiB avail; 24 KiB/s rd, 2.0 KiB/s wr, 31 op/s; 21/486 objects degraded (4.321%) 2024-03-07T03:23:43.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:43 smithi195 bash[66542]: audit 2024-03-07T03:23:42.434465+0000 mon.a (mon.0) 883 : audit 0 from='client.? 172.21.15.148:0/3648191181' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:44.788 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:44 smithi148 bash[79819]: cluster 2024-03-07T03:23:44.409435+0000 mon.a (mon.0) 884 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:23:44.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:44 smithi148 bash[82222]: cluster 2024-03-07T03:23:44.409435+0000 mon.a (mon.0) 884 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:23:44.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:44 smithi195 bash[66542]: cluster 2024-03-07T03:23:44.409435+0000 mon.a (mon.0) 884 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-03-07T03:23:45.289 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:44 smithi148 bash[99646]: debug 2024-03-07T03:23:44.990+0000 7f73d7981700 1 -- 172.21.15.148:0/3551135292 <== mon.0 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564069ea7680 con 0x564069ee2c00 2024-03-07T03:23:45.789 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:45 smithi148 bash[99646]: debug 2024-03-07T03:23:45.546+0000 7f73e0b197c0 -1 Falling back to public interface 2024-03-07T03:23:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:45 smithi148 bash[79819]: cluster 2024-03-07T03:23:44.356359+0000 mgr.y (mgr.34101) 280 : cluster 0 pgmap v544: 361 pgs: 95 active+undersized, 39 active+undersized+degraded, 227 active+clean; 2.6 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 403 KiB/s wr, 24 op/s; 62/498 objects degraded (12.450%) 2024-03-07T03:23:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:45 smithi148 bash[79819]: audit 2024-03-07T03:23:44.794689+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.148:0/194472017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:45.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:45 smithi148 bash[82222]: cluster 2024-03-07T03:23:44.356359+0000 mgr.y (mgr.34101) 280 : cluster 0 pgmap v544: 361 pgs: 95 active+undersized, 39 active+undersized+degraded, 227 active+clean; 2.6 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 403 KiB/s wr, 24 op/s; 62/498 objects degraded (12.450%) 2024-03-07T03:23:45.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:45 smithi148 bash[82222]: audit 2024-03-07T03:23:44.794689+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.148:0/194472017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:45 smithi195 bash[66542]: cluster 2024-03-07T03:23:44.356359+0000 mgr.y (mgr.34101) 280 : cluster 0 pgmap v544: 361 pgs: 95 active+undersized, 39 active+undersized+degraded, 227 active+clean; 2.6 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 403 KiB/s wr, 24 op/s; 62/498 objects degraded (12.450%) 2024-03-07T03:23:45.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:45 smithi195 bash[66542]: audit 2024-03-07T03:23:44.794689+0000 mon.c (mon.2) 41 : audit 0 from='client.? 172.21.15.148:0/194472017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:46.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:46 smithi148 bash[79819]: cluster 2024-03-07T03:23:46.463928+0000 mon.a (mon.0) 885 : cluster 3 Health check update: Degraded data redundancy: 63/504 objects degraded (12.500%), 39 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:46.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:46 smithi148 bash[82222]: cluster 2024-03-07T03:23:46.463928+0000 mon.a (mon.0) 885 : cluster 3 Health check update: Degraded data redundancy: 63/504 objects degraded (12.500%), 39 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:46.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:46 smithi195 bash[66542]: cluster 2024-03-07T03:23:46.463928+0000 mon.a (mon.0) 885 : cluster 3 Health check update: Degraded data redundancy: 63/504 objects degraded (12.500%), 39 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:47.502 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:23:47] "GET /metrics HTTP/1.1" 200 55797 "" "Prometheus/2.43.0" 2024-03-07T03:23:47.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:47 smithi148 bash[82222]: cluster 2024-03-07T03:23:46.358402+0000 mgr.y (mgr.34101) 281 : cluster 0 pgmap v545: 361 pgs: 95 active+undersized, 39 active+undersized+degraded, 227 active+clean; 2.6 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 340 KiB/s wr, 44 op/s; 63/504 objects degraded (12.500%) 2024-03-07T03:23:47.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:47 smithi148 bash[82222]: audit 2024-03-07T03:23:47.168762+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.148:0/3548521470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:47.788 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:47 smithi148 bash[82222]: cluster 2024-03-07T03:23:47.223463+0000 mon.a (mon.0) 886 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2024-03-07T03:23:47.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:47 smithi148 bash[79819]: cluster 2024-03-07T03:23:46.358402+0000 mgr.y (mgr.34101) 281 : cluster 0 pgmap v545: 361 pgs: 95 active+undersized, 39 active+undersized+degraded, 227 active+clean; 2.6 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 340 KiB/s wr, 44 op/s; 63/504 objects degraded (12.500%) 2024-03-07T03:23:47.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:47 smithi148 bash[79819]: audit 2024-03-07T03:23:47.168762+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.148:0/3548521470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:47.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:47 smithi148 bash[79819]: cluster 2024-03-07T03:23:47.223463+0000 mon.a (mon.0) 886 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2024-03-07T03:23:47.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:47 smithi195 bash[66542]: cluster 2024-03-07T03:23:46.358402+0000 mgr.y (mgr.34101) 281 : cluster 0 pgmap v545: 361 pgs: 95 active+undersized, 39 active+undersized+degraded, 227 active+clean; 2.6 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 340 KiB/s wr, 44 op/s; 63/504 objects degraded (12.500%) 2024-03-07T03:23:47.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:47 smithi195 bash[66542]: audit 2024-03-07T03:23:47.168762+0000 mon.c (mon.2) 42 : audit 0 from='client.? 172.21.15.148:0/3548521470' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:47.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:47 smithi195 bash[66542]: cluster 2024-03-07T03:23:47.223463+0000 mon.a (mon.0) 886 : cluster 0 osdmap e632: 8 total, 7 up, 8 in 2024-03-07T03:23:49.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[82222]: audit 2024-03-07T03:23:47.771409+0000 mon.a (mon.0) 887 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[82222]: audit 2024-03-07T03:23:47.778102+0000 mon.a (mon.0) 888 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[82222]: audit 2024-03-07T03:23:48.029065+0000 mon.a (mon.0) 889 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[82222]: audit 2024-03-07T03:23:48.040274+0000 mon.a (mon.0) 890 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[82222]: cluster 2024-03-07T03:23:48.360080+0000 mgr.y (mgr.34101) 282 : cluster 0 pgmap v547: 361 pgs: 94 active+undersized, 40 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 872 KiB/s wr, 18 op/s; 64/507 objects degraded (12.623%) 2024-03-07T03:23:49.040 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[99646]: debug 2024-03-07T03:23:48.786+0000 7f73e0b197c0 -1 osd.2 627 log_to_monitors true 2024-03-07T03:23:49.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[79819]: audit 2024-03-07T03:23:47.771409+0000 mon.a (mon.0) 887 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[79819]: audit 2024-03-07T03:23:47.778102+0000 mon.a (mon.0) 888 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[79819]: audit 2024-03-07T03:23:48.029065+0000 mon.a (mon.0) 889 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[79819]: audit 2024-03-07T03:23:48.040274+0000 mon.a (mon.0) 890 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:48 smithi148 bash[79819]: cluster 2024-03-07T03:23:48.360080+0000 mgr.y (mgr.34101) 282 : cluster 0 pgmap v547: 361 pgs: 94 active+undersized, 40 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 872 KiB/s wr, 18 op/s; 64/507 objects degraded (12.623%) 2024-03-07T03:23:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:48 smithi195 bash[66542]: audit 2024-03-07T03:23:47.771409+0000 mon.a (mon.0) 887 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:48 smithi195 bash[66542]: audit 2024-03-07T03:23:47.778102+0000 mon.a (mon.0) 888 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:48 smithi195 bash[66542]: audit 2024-03-07T03:23:48.029065+0000 mon.a (mon.0) 889 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:48 smithi195 bash[66542]: audit 2024-03-07T03:23:48.040274+0000 mon.a (mon.0) 890 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:48 smithi195 bash[66542]: cluster 2024-03-07T03:23:48.360080+0000 mgr.y (mgr.34101) 282 : cluster 0 pgmap v547: 361 pgs: 94 active+undersized, 40 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 445 MiB used, 715 GiB / 715 GiB avail; 8.6 KiB/s rd, 872 KiB/s wr, 18 op/s; 64/507 objects degraded (12.623%) 2024-03-07T03:23:50.039 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[99646]: debug 2024-03-07T03:23:49.882+0000 7f73d1f71700 -1 osd.2 627 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:23:50.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[79819]: audit 2024-03-07T03:23:48.794356+0000 mon.c (mon.2) 43 : audit 1 from='osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:23:50.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[79819]: audit 2024-03-07T03:23:48.794664+0000 mon.a (mon.0) 891 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:23:50.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[79819]: audit 2024-03-07T03:23:48.878588+0000 mon.a (mon.0) 892 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:50.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[79819]: audit 2024-03-07T03:23:48.892693+0000 mon.a (mon.0) 893 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:50.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[79819]: audit 2024-03-07T03:23:49.502834+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.148:0/3487085405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:50.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[82222]: audit 2024-03-07T03:23:48.794356+0000 mon.c (mon.2) 43 : audit 1 from='osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:23:50.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[82222]: audit 2024-03-07T03:23:48.794664+0000 mon.a (mon.0) 891 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:23:50.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[82222]: audit 2024-03-07T03:23:48.878588+0000 mon.a (mon.0) 892 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:50.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[82222]: audit 2024-03-07T03:23:48.892693+0000 mon.a (mon.0) 893 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:50.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:49 smithi148 bash[82222]: audit 2024-03-07T03:23:49.502834+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.148:0/3487085405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:50.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:49 smithi195 bash[66542]: audit 2024-03-07T03:23:48.794356+0000 mon.c (mon.2) 43 : audit 1 from='osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:23:50.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:49 smithi195 bash[66542]: audit 2024-03-07T03:23:48.794664+0000 mon.a (mon.0) 891 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-07T03:23:50.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:49 smithi195 bash[66542]: audit 2024-03-07T03:23:48.878588+0000 mon.a (mon.0) 892 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:50.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:49 smithi195 bash[66542]: audit 2024-03-07T03:23:48.892693+0000 mon.a (mon.0) 893 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:50.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:49 smithi195 bash[66542]: audit 2024-03-07T03:23:49.502834+0000 mon.c (mon.2) 44 : audit 0 from='client.? 172.21.15.148:0/3487085405' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:51.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[82222]: audit 2024-03-07T03:23:49.787388+0000 mon.a (mon.0) 894 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:23:51.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[82222]: cluster 2024-03-07T03:23:49.796182+0000 mon.a (mon.0) 895 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2024-03-07T03:23:51.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[82222]: audit 2024-03-07T03:23:49.800247+0000 mon.c (mon.2) 45 : audit 1 from='osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:23:51.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[82222]: audit 2024-03-07T03:23:49.801985+0000 mon.a (mon.0) 896 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:23:51.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[82222]: cluster 2024-03-07T03:23:50.361251+0000 mgr.y (mgr.34101) 283 : cluster 0 pgmap v549: 361 pgs: 94 active+undersized, 40 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 430 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 769 KiB/s wr, 17 op/s; 64/507 objects degraded (12.623%) 2024-03-07T03:23:51.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[79819]: debug 2024-03-07T03:23:50.786+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[79819]: audit 2024-03-07T03:23:49.787388+0000 mon.a (mon.0) 894 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:23:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[79819]: cluster 2024-03-07T03:23:49.796182+0000 mon.a (mon.0) 895 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2024-03-07T03:23:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[79819]: audit 2024-03-07T03:23:49.800247+0000 mon.c (mon.2) 45 : audit 1 from='osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:23:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[79819]: audit 2024-03-07T03:23:49.801985+0000 mon.a (mon.0) 896 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:23:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:50 smithi148 bash[79819]: cluster 2024-03-07T03:23:50.361251+0000 mgr.y (mgr.34101) 283 : cluster 0 pgmap v549: 361 pgs: 94 active+undersized, 40 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 430 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 769 KiB/s wr, 17 op/s; 64/507 objects degraded (12.623%) 2024-03-07T03:23:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:50 smithi195 bash[66542]: audit 2024-03-07T03:23:49.787388+0000 mon.a (mon.0) 894 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-07T03:23:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:50 smithi195 bash[66542]: cluster 2024-03-07T03:23:49.796182+0000 mon.a (mon.0) 895 : cluster 0 osdmap e633: 8 total, 7 up, 8 in 2024-03-07T03:23:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:50 smithi195 bash[66542]: audit 2024-03-07T03:23:49.800247+0000 mon.c (mon.2) 45 : audit 1 from='osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:23:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:50 smithi195 bash[66542]: audit 2024-03-07T03:23:49.801985+0000 mon.a (mon.0) 896 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:23:51.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:50 smithi195 bash[66542]: cluster 2024-03-07T03:23:50.361251+0000 mgr.y (mgr.34101) 283 : cluster 0 pgmap v549: 361 pgs: 94 active+undersized, 40 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 430 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 769 KiB/s wr, 17 op/s; 64/507 objects degraded (12.623%) 2024-03-07T03:23:52.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:51 smithi195 bash[66542]: cluster 2024-03-07T03:23:50.789086+0000 mon.a (mon.0) 897 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:52.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:51 smithi195 bash[66542]: cluster 2024-03-07T03:23:50.789166+0000 mon.a (mon.0) 898 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:23:52.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:51 smithi195 bash[66542]: cluster 2024-03-07T03:23:50.799213+0000 mon.a (mon.0) 899 : cluster 1 osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140] boot 2024-03-07T03:23:52.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:51 smithi195 bash[66542]: cluster 2024-03-07T03:23:50.799397+0000 mon.a (mon.0) 900 : cluster 0 osdmap e634: 8 total, 8 up, 8 in 2024-03-07T03:23:52.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:51 smithi195 bash[66542]: audit 2024-03-07T03:23:50.800498+0000 mon.a (mon.0) 901 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:23:52.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:51 smithi195 bash[66542]: cluster 2024-03-07T03:23:51.800641+0000 mon.a (mon.0) 902 : cluster 0 osdmap e635: 8 total, 8 up, 8 in 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[82222]: cluster 2024-03-07T03:23:50.789086+0000 mon.a (mon.0) 897 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[82222]: cluster 2024-03-07T03:23:50.789166+0000 mon.a (mon.0) 898 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[82222]: cluster 2024-03-07T03:23:50.799213+0000 mon.a (mon.0) 899 : cluster 1 osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140] boot 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[82222]: cluster 2024-03-07T03:23:50.799397+0000 mon.a (mon.0) 900 : cluster 0 osdmap e634: 8 total, 8 up, 8 in 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[82222]: audit 2024-03-07T03:23:50.800498+0000 mon.a (mon.0) 901 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[82222]: cluster 2024-03-07T03:23:51.800641+0000 mon.a (mon.0) 902 : cluster 0 osdmap e635: 8 total, 8 up, 8 in 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[79819]: cluster 2024-03-07T03:23:50.789086+0000 mon.a (mon.0) 897 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:23:52.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[79819]: cluster 2024-03-07T03:23:50.789166+0000 mon.a (mon.0) 898 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:23:52.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[79819]: cluster 2024-03-07T03:23:50.799213+0000 mon.a (mon.0) 899 : cluster 1 osd.2 [v2:172.21.15.148:6818/2867443140,v1:172.21.15.148:6819/2867443140] boot 2024-03-07T03:23:52.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[79819]: cluster 2024-03-07T03:23:50.799397+0000 mon.a (mon.0) 900 : cluster 0 osdmap e634: 8 total, 8 up, 8 in 2024-03-07T03:23:52.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[79819]: audit 2024-03-07T03:23:50.800498+0000 mon.a (mon.0) 901 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-07T03:23:52.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:51 smithi148 bash[79819]: cluster 2024-03-07T03:23:51.800641+0000 mon.a (mon.0) 902 : cluster 0 osdmap e635: 8 total, 8 up, 8 in 2024-03-07T03:23:52.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:23:52 smithi195 bash[59134]: ts=2024-03-07T03:23:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:23:53.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:52 smithi148 bash[79819]: audit 2024-03-07T03:23:51.914295+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.148:0/2833741344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:53.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:52 smithi148 bash[79819]: cluster 2024-03-07T03:23:52.219206+0000 mon.a (mon.0) 903 : cluster 3 Health check update: Degraded data redundancy: 64/507 objects degraded (12.623%), 40 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:53.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:52 smithi148 bash[79819]: cluster 2024-03-07T03:23:52.362564+0000 mgr.y (mgr.34101) 284 : cluster 0 pgmap v552: 361 pgs: 17 peering, 83 active+undersized, 34 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 434 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 797 KiB/s wr, 2 op/s; 54/507 objects degraded (10.651%) 2024-03-07T03:23:53.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:52 smithi148 bash[82222]: audit 2024-03-07T03:23:51.914295+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.148:0/2833741344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:53.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:52 smithi148 bash[82222]: cluster 2024-03-07T03:23:52.219206+0000 mon.a (mon.0) 903 : cluster 3 Health check update: Degraded data redundancy: 64/507 objects degraded (12.623%), 40 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:53.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:52 smithi148 bash[82222]: cluster 2024-03-07T03:23:52.362564+0000 mgr.y (mgr.34101) 284 : cluster 0 pgmap v552: 361 pgs: 17 peering, 83 active+undersized, 34 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 434 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 797 KiB/s wr, 2 op/s; 54/507 objects degraded (10.651%) 2024-03-07T03:23:53.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:52 smithi195 bash[66542]: audit 2024-03-07T03:23:51.914295+0000 mon.c (mon.2) 46 : audit 0 from='client.? 172.21.15.148:0/2833741344' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:53.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:52 smithi195 bash[66542]: cluster 2024-03-07T03:23:52.219206+0000 mon.a (mon.0) 903 : cluster 3 Health check update: Degraded data redundancy: 64/507 objects degraded (12.623%), 40 pgs degraded (PG_DEGRADED) 2024-03-07T03:23:53.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:52 smithi195 bash[66542]: cluster 2024-03-07T03:23:52.362564+0000 mgr.y (mgr.34101) 284 : cluster 0 pgmap v552: 361 pgs: 17 peering, 83 active+undersized, 34 active+undersized+degraded, 227 active+clean; 6.6 MiB data, 434 MiB used, 715 GiB / 715 GiB avail; 995 B/s rd, 797 KiB/s wr, 2 op/s; 54/507 objects degraded (10.651%) 2024-03-07T03:23:54.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:54 smithi195 bash[66542]: audit 2024-03-07T03:23:54.271051+0000 mon.a (mon.0) 904 : audit 0 from='client.? 172.21.15.148:0/412790163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:54.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:54 smithi148 bash[79819]: audit 2024-03-07T03:23:54.271051+0000 mon.a (mon.0) 904 : audit 0 from='client.? 172.21.15.148:0/412790163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:54.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:54 smithi148 bash[82222]: audit 2024-03-07T03:23:54.271051+0000 mon.a (mon.0) 904 : audit 0 from='client.? 172.21.15.148:0/412790163' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:55.630 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:55 smithi148 bash[79819]: cluster 2024-03-07T03:23:54.364921+0000 mgr.y (mgr.34101) 285 : cluster 0 pgmap v553: 361 pgs: 17 peering, 38 active+undersized, 14 active+undersized+degraded, 292 active+clean; 9.6 MiB data, 428 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 512 KiB/s wr, 1 op/s; 22/510 objects degraded (4.314%); 0 B/s, 0 objects/s recovering 2024-03-07T03:23:55.631 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:55 smithi148 bash[82222]: cluster 2024-03-07T03:23:54.364921+0000 mgr.y (mgr.34101) 285 : cluster 0 pgmap v553: 361 pgs: 17 peering, 38 active+undersized, 14 active+undersized+degraded, 292 active+clean; 9.6 MiB data, 428 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 512 KiB/s wr, 1 op/s; 22/510 objects degraded (4.314%); 0 B/s, 0 objects/s recovering 2024-03-07T03:23:55.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:55 smithi195 bash[66542]: cluster 2024-03-07T03:23:54.364921+0000 mgr.y (mgr.34101) 285 : cluster 0 pgmap v553: 361 pgs: 17 peering, 38 active+undersized, 14 active+undersized+degraded, 292 active+clean; 9.6 MiB data, 428 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 512 KiB/s wr, 1 op/s; 22/510 objects degraded (4.314%); 0 B/s, 0 objects/s recovering 2024-03-07T03:23:56.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:56 smithi148 bash[79819]: cluster 2024-03-07T03:23:56.419438+0000 mon.a (mon.0) 905 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/510 objects degraded (4.314%), 14 pgs degraded) 2024-03-07T03:23:56.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:56 smithi148 bash[82222]: cluster 2024-03-07T03:23:56.419438+0000 mon.a (mon.0) 905 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/510 objects degraded (4.314%), 14 pgs degraded) 2024-03-07T03:23:56.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:56 smithi195 bash[66542]: cluster 2024-03-07T03:23:56.419438+0000 mon.a (mon.0) 905 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22/510 objects degraded (4.314%), 14 pgs degraded) 2024-03-07T03:23:57.447 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.RemoveFullTry (20791 ms) 2024-03-07T03:23:57.448 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-03-07T03:23:57.458 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:23:57] "GET /metrics HTTP/1.1" 200 55797 "" "Prometheus/2.43.0" 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[79819]: cluster 2024-03-07T03:23:56.366741+0000 mgr.y (mgr.34101) 286 : cluster 0 pgmap v554: 361 pgs: 361 active+clean; 12 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 780 KiB/s wr, 4 op/s; 1.2 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[79819]: audit 2024-03-07T03:23:56.630084+0000 mon.a (mon.0) 906 : audit 0 from='client.? 172.21.15.148:0/3321823782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[79819]: cluster 2024-03-07T03:23:57.220571+0000 mon.a (mon.0) 907 : cluster 3 pool 'test-librbd-smithi148-68562-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[79819]: cluster 2024-03-07T03:23:57.220951+0000 mon.a (mon.0) 908 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[79819]: cluster 2024-03-07T03:23:57.229654+0000 mon.a (mon.0) 909 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[79819]: audit 2024-03-07T03:23:57.318973+0000 mon.a (mon.0) 910 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[79819]: audit 2024-03-07T03:23:57.320365+0000 mon.a (mon.0) 911 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[82222]: cluster 2024-03-07T03:23:56.366741+0000 mgr.y (mgr.34101) 286 : cluster 0 pgmap v554: 361 pgs: 361 active+clean; 12 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 780 KiB/s wr, 4 op/s; 1.2 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[82222]: audit 2024-03-07T03:23:56.630084+0000 mon.a (mon.0) 906 : audit 0 from='client.? 172.21.15.148:0/3321823782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:57.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[82222]: cluster 2024-03-07T03:23:57.220571+0000 mon.a (mon.0) 907 : cluster 3 pool 'test-librbd-smithi148-68562-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:23:57.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[82222]: cluster 2024-03-07T03:23:57.220951+0000 mon.a (mon.0) 908 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:23:57.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[82222]: cluster 2024-03-07T03:23:57.229654+0000 mon.a (mon.0) 909 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2024-03-07T03:23:57.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[82222]: audit 2024-03-07T03:23:57.318973+0000 mon.a (mon.0) 910 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:57.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:57 smithi148 bash[82222]: audit 2024-03-07T03:23:57.320365+0000 mon.a (mon.0) 911 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:57 smithi195 bash[66542]: cluster 2024-03-07T03:23:56.366741+0000 mgr.y (mgr.34101) 286 : cluster 0 pgmap v554: 361 pgs: 361 active+clean; 12 MiB data, 436 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 780 KiB/s wr, 4 op/s; 1.2 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:23:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:57 smithi195 bash[66542]: audit 2024-03-07T03:23:56.630084+0000 mon.a (mon.0) 906 : audit 0 from='client.? 172.21.15.148:0/3321823782' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:57 smithi195 bash[66542]: cluster 2024-03-07T03:23:57.220571+0000 mon.a (mon.0) 907 : cluster 3 pool 'test-librbd-smithi148-68562-23' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:23:57.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:57 smithi195 bash[66542]: cluster 2024-03-07T03:23:57.220951+0000 mon.a (mon.0) 908 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:23:57.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:57 smithi195 bash[66542]: cluster 2024-03-07T03:23:57.229654+0000 mon.a (mon.0) 909 : cluster 0 osdmap e636: 8 total, 8 up, 8 in 2024-03-07T03:23:57.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:57 smithi195 bash[66542]: audit 2024-03-07T03:23:57.318973+0000 mon.a (mon.0) 910 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:57.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:57 smithi195 bash[66542]: audit 2024-03-07T03:23:57.320365+0000 mon.a (mon.0) 911 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:23:59.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:58 smithi148 bash[72584]: debug 2024-03-07T03:23:58.789+0000 7f6cc280d700 -1 mgr.server reply reply (11) Resource temporarily unavailable 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:23:59.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:23:58 smithi148 bash[72584]: debug 2024-03-07T03:23:58.789+0000 7f6cc280d700 -1 mgr.server reply reply (16) Device or resource busy 32 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-03-07T03:23:59.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: cluster 2024-03-07T03:23:58.368039+0000 mgr.y (mgr.34101) 287 : cluster 0 pgmap v556: 361 pgs: 361 active+clean; 8.6 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 1.2 MiB/s wr, 17 op/s; 1.1 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:23:59.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: cluster 2024-03-07T03:23:58.499870+0000 mon.a (mon.0) 912 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.508430+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.148:0/2941146867' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.509788+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.688426+0000 mon.a (mon.0) 914 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.697382+0000 mon.a (mon.0) 915 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.698923+0000 mon.a (mon.0) 916 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.699911+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.707924+0000 mon.a (mon.0) 918 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.786476+0000 mon.a (mon.0) 919 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.788491+0000 mon.a (mon.0) 920 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.789989+0000 mon.a (mon.0) 921 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.791022+0000 mon.a (mon.0) 922 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.792372+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[82222]: audit 2024-03-07T03:23:58.984206+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.148:0/3336416361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: cluster 2024-03-07T03:23:58.368039+0000 mgr.y (mgr.34101) 287 : cluster 0 pgmap v556: 361 pgs: 361 active+clean; 8.6 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 1.2 MiB/s wr, 17 op/s; 1.1 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: cluster 2024-03-07T03:23:58.499870+0000 mon.a (mon.0) 912 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.508430+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.148:0/2941146867' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.509788+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.688426+0000 mon.a (mon.0) 914 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.697382+0000 mon.a (mon.0) 915 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.698923+0000 mon.a (mon.0) 916 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.699911+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:23:59.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.707924+0000 mon.a (mon.0) 918 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.786476+0000 mon.a (mon.0) 919 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:59.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.788491+0000 mon.a (mon.0) 920 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.789989+0000 mon.a (mon.0) 921 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.791022+0000 mon.a (mon.0) 922 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.792372+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:23:59.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:23:59 smithi148 bash[79819]: audit 2024-03-07T03:23:58.984206+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.148:0/3336416361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: cluster 2024-03-07T03:23:58.368039+0000 mgr.y (mgr.34101) 287 : cluster 0 pgmap v556: 361 pgs: 361 active+clean; 8.6 MiB data, 431 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 1.2 MiB/s wr, 17 op/s; 1.1 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: cluster 2024-03-07T03:23:58.499870+0000 mon.a (mon.0) 912 : cluster 0 osdmap e637: 8 total, 8 up, 8 in 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.508430+0000 mon.c (mon.2) 47 : audit 1 from='client.? 172.21.15.148:0/2941146867' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.509788+0000 mon.a (mon.0) 913 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.688426+0000 mon.a (mon.0) 914 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.697382+0000 mon.a (mon.0) 915 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.698923+0000 mon.a (mon.0) 916 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.699911+0000 mon.a (mon.0) 917 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.707924+0000 mon.a (mon.0) 918 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.786476+0000 mon.a (mon.0) 919 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.788491+0000 mon.a (mon.0) 920 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.789989+0000 mon.a (mon.0) 921 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.791022+0000 mon.a (mon.0) 922 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:23:59.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.792372+0000 mon.a (mon.0) 923 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:23:59.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:23:59 smithi195 bash[66542]: audit 2024-03-07T03:23:58.984206+0000 mon.c (mon.2) 48 : audit 0 from='client.? 172.21.15.148:0/3336416361' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[79819]: audit 2024-03-07T03:23:58.792549+0000 mgr.y (mgr.34101) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[79819]: cephadm 2024-03-07T03:23:58.793720+0000 mgr.y (mgr.34101) 289 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:24:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[79819]: audit 2024-03-07T03:23:59.489090+0000 mon.a (mon.0) 924 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:24:00.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[79819]: cluster 2024-03-07T03:23:59.498054+0000 mon.a (mon.0) 925 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2024-03-07T03:24:00.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[82222]: audit 2024-03-07T03:23:58.792549+0000 mgr.y (mgr.34101) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:00.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[82222]: cephadm 2024-03-07T03:23:58.793720+0000 mgr.y (mgr.34101) 289 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:24:00.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[82222]: audit 2024-03-07T03:23:59.489090+0000 mon.a (mon.0) 924 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:24:00.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:00 smithi148 bash[82222]: cluster 2024-03-07T03:23:59.498054+0000 mon.a (mon.0) 925 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2024-03-07T03:24:00.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:00 smithi195 bash[66542]: audit 2024-03-07T03:23:58.792549+0000 mgr.y (mgr.34101) 288 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:00.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:00 smithi195 bash[66542]: cephadm 2024-03-07T03:23:58.793720+0000 mgr.y (mgr.34101) 289 : cephadm 1 Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-03-07T03:24:00.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:00 smithi195 bash[66542]: audit 2024-03-07T03:23:59.489090+0000 mon.a (mon.0) 924 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:24:00.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:00 smithi195 bash[66542]: cluster 2024-03-07T03:23:59.498054+0000 mon.a (mon.0) 925 : cluster 0 osdmap e638: 8 total, 8 up, 8 in 2024-03-07T03:24:01.574 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:24:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:01 smithi195 bash[66542]: cluster 2024-03-07T03:24:00.369716+0000 mgr.y (mgr.34101) 290 : cluster 0 pgmap v559: 393 pgs: 25 unknown, 368 active+clean; 584 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 1.0 MiB/s wr, 29 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:24:01.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:01 smithi195 bash[66542]: cluster 2024-03-07T03:24:00.529581+0000 mon.a (mon.0) 926 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2024-03-07T03:24:01.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:01 smithi195 bash[66542]: audit 2024-03-07T03:24:00.544661+0000 mon.a (mon.0) 927 : audit 1 from='client.? 172.21.15.148:0/820436084' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:24:01.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:01 smithi195 bash[66542]: audit 2024-03-07T03:24:01.341974+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.148:0/2377009168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:02.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[72584]: [07/Mar/2024:03:24:01] ENGINE Bus STOPPING 2024-03-07T03:24:02.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[72584]: [07/Mar/2024:03:24:01] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:24:02.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[72584]: [07/Mar/2024:03:24:01] ENGINE Bus STOPPED 2024-03-07T03:24:02.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[72584]: [07/Mar/2024:03:24:01] ENGINE Bus STARTING 2024-03-07T03:24:02.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[72584]: [07/Mar/2024:03:24:01] ENGINE Serving on http://:::9283 2024-03-07T03:24:02.039 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[72584]: [07/Mar/2024:03:24:01] ENGINE Bus STARTED 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[79819]: cluster 2024-03-07T03:24:00.369716+0000 mgr.y (mgr.34101) 290 : cluster 0 pgmap v559: 393 pgs: 25 unknown, 368 active+clean; 584 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 1.0 MiB/s wr, 29 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[79819]: cluster 2024-03-07T03:24:00.529581+0000 mon.a (mon.0) 926 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[79819]: audit 2024-03-07T03:24:00.544661+0000 mon.a (mon.0) 927 : audit 1 from='client.? 172.21.15.148:0/820436084' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[79819]: audit 2024-03-07T03:24:01.341974+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.148:0/2377009168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[82222]: cluster 2024-03-07T03:24:00.369716+0000 mgr.y (mgr.34101) 290 : cluster 0 pgmap v559: 393 pgs: 25 unknown, 368 active+clean; 584 KiB data, 419 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 1.0 MiB/s wr, 29 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[82222]: cluster 2024-03-07T03:24:00.529581+0000 mon.a (mon.0) 926 : cluster 0 osdmap e639: 8 total, 8 up, 8 in 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[82222]: audit 2024-03-07T03:24:00.544661+0000 mon.a (mon.0) 927 : audit 1 from='client.? 172.21.15.148:0/820436084' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-25","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:24:02.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:01 smithi148 bash[82222]: audit 2024-03-07T03:24:01.341974+0000 mon.c (mon.2) 49 : audit 0 from='client.? 172.21.15.148:0/2377009168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:02.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:02 smithi195 bash[59134]: ts=2024-03-07T03:24:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:02 smithi195 bash[66542]: audit 2024-03-07T03:24:01.522787+0000 mon.a (mon.0) 928 : audit 1 from='client.? 172.21.15.148:0/820436084' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:24:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:02 smithi195 bash[66542]: cluster 2024-03-07T03:24:01.530426+0000 mon.a (mon.0) 929 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2024-03-07T03:24:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:02 smithi195 bash[66542]: audit 2024-03-07T03:24:01.583187+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:24:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:02 smithi195 bash[66542]: cluster 2024-03-07T03:24:02.222083+0000 mon.a (mon.0) 931 : cluster 1 pool 'test-librbd-smithi148-68562-23' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:24:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:02 smithi195 bash[66542]: cluster 2024-03-07T03:24:02.222997+0000 mon.a (mon.0) 932 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:24:02.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:02 smithi195 bash[66542]: cluster 2024-03-07T03:24:02.236450+0000 mon.a (mon.0) 933 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2024-03-07T03:24:03.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[79819]: audit 2024-03-07T03:24:01.522787+0000 mon.a (mon.0) 928 : audit 1 from='client.? 172.21.15.148:0/820436084' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:24:03.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[79819]: cluster 2024-03-07T03:24:01.530426+0000 mon.a (mon.0) 929 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2024-03-07T03:24:03.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[79819]: audit 2024-03-07T03:24:01.583187+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:24:03.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[79819]: cluster 2024-03-07T03:24:02.222083+0000 mon.a (mon.0) 931 : cluster 1 pool 'test-librbd-smithi148-68562-23' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[79819]: cluster 2024-03-07T03:24:02.222997+0000 mon.a (mon.0) 932 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[79819]: cluster 2024-03-07T03:24:02.236450+0000 mon.a (mon.0) 933 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[82222]: audit 2024-03-07T03:24:01.522787+0000 mon.a (mon.0) 928 : audit 1 from='client.? 172.21.15.148:0/820436084' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[82222]: cluster 2024-03-07T03:24:01.530426+0000 mon.a (mon.0) 929 : cluster 0 osdmap e640: 8 total, 8 up, 8 in 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[82222]: audit 2024-03-07T03:24:01.583187+0000 mon.a (mon.0) 930 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[82222]: cluster 2024-03-07T03:24:02.222083+0000 mon.a (mon.0) 931 : cluster 1 pool 'test-librbd-smithi148-68562-23' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[82222]: cluster 2024-03-07T03:24:02.222997+0000 mon.a (mon.0) 932 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:24:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:02 smithi148 bash[82222]: cluster 2024-03-07T03:24:02.236450+0000 mon.a (mon.0) 933 : cluster 0 osdmap e641: 8 total, 8 up, 8 in 2024-03-07T03:24:04.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[79819]: cluster 2024-03-07T03:24:02.370745+0000 mgr.y (mgr.34101) 291 : cluster 0 pgmap v563: 425 pgs: 5 creating+peering, 48 unknown, 372 active+clean; 584 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.0 KiB/s wr, 12 op/s 2024-03-07T03:24:04.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[79819]: audit 2024-03-07T03:24:02.587119+0000 mon.a (mon.0) 934 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:04.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[79819]: cluster 2024-03-07T03:24:03.248913+0000 mon.a (mon.0) 935 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2024-03-07T03:24:04.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[79819]: audit 2024-03-07T03:24:03.294325+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:24:04.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[79819]: audit 2024-03-07T03:24:03.294926+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:24:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[82222]: cluster 2024-03-07T03:24:02.370745+0000 mgr.y (mgr.34101) 291 : cluster 0 pgmap v563: 425 pgs: 5 creating+peering, 48 unknown, 372 active+clean; 584 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.0 KiB/s wr, 12 op/s 2024-03-07T03:24:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[82222]: audit 2024-03-07T03:24:02.587119+0000 mon.a (mon.0) 934 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[82222]: cluster 2024-03-07T03:24:03.248913+0000 mon.a (mon.0) 935 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2024-03-07T03:24:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[82222]: audit 2024-03-07T03:24:03.294325+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:24:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:03 smithi148 bash[82222]: audit 2024-03-07T03:24:03.294926+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:24:04.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:03 smithi195 bash[66542]: cluster 2024-03-07T03:24:02.370745+0000 mgr.y (mgr.34101) 291 : cluster 0 pgmap v563: 425 pgs: 5 creating+peering, 48 unknown, 372 active+clean; 584 KiB data, 415 MiB used, 715 GiB / 715 GiB avail; 7.5 KiB/s rd, 1.0 KiB/s wr, 12 op/s 2024-03-07T03:24:04.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:03 smithi195 bash[66542]: audit 2024-03-07T03:24:02.587119+0000 mon.a (mon.0) 934 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:04.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:03 smithi195 bash[66542]: cluster 2024-03-07T03:24:03.248913+0000 mon.a (mon.0) 935 : cluster 0 osdmap e642: 8 total, 8 up, 8 in 2024-03-07T03:24:04.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:03 smithi195 bash[66542]: audit 2024-03-07T03:24:03.294325+0000 mon.c (mon.2) 50 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:24:04.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:03 smithi195 bash[66542]: audit 2024-03-07T03:24:03.294926+0000 mon.a (mon.0) 936 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]: dispatch 2024-03-07T03:24:05.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:04 smithi148 bash[79819]: audit 2024-03-07T03:24:03.700106+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.148:0/635740651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:05.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:04 smithi148 bash[79819]: audit 2024-03-07T03:24:04.239291+0000 mon.a (mon.0) 937 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:24:05.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:04 smithi148 bash[79819]: cluster 2024-03-07T03:24:04.248893+0000 mon.a (mon.0) 938 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2024-03-07T03:24:05.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:04 smithi148 bash[82222]: audit 2024-03-07T03:24:03.700106+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.148:0/635740651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:05.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:04 smithi148 bash[82222]: audit 2024-03-07T03:24:04.239291+0000 mon.a (mon.0) 937 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:24:05.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:04 smithi148 bash[82222]: cluster 2024-03-07T03:24:04.248893+0000 mon.a (mon.0) 938 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2024-03-07T03:24:05.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:04 smithi195 bash[66542]: audit 2024-03-07T03:24:03.700106+0000 mon.c (mon.2) 51 : audit 0 from='client.? 172.21.15.148:0/635740651' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:05.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:04 smithi195 bash[66542]: audit 2024-03-07T03:24:04.239291+0000 mon.a (mon.0) 937 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi148-68562-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-03-07T03:24:05.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:04 smithi195 bash[66542]: cluster 2024-03-07T03:24:04.248893+0000 mon.a (mon.0) 938 : cluster 0 osdmap e643: 8 total, 8 up, 8 in 2024-03-07T03:24:06.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:05 smithi148 bash[79819]: cluster 2024-03-07T03:24:04.371716+0000 mgr.y (mgr.34101) 292 : cluster 0 pgmap v566: 425 pgs: 5 creating+peering, 4 unknown, 416 active+clean; 584 KiB data, 405 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-03-07T03:24:06.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:05 smithi148 bash[82222]: cluster 2024-03-07T03:24:04.371716+0000 mgr.y (mgr.34101) 292 : cluster 0 pgmap v566: 425 pgs: 5 creating+peering, 4 unknown, 416 active+clean; 584 KiB data, 405 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-03-07T03:24:06.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:05 smithi195 bash[66542]: cluster 2024-03-07T03:24:04.371716+0000 mgr.y (mgr.34101) 292 : cluster 0 pgmap v566: 425 pgs: 5 creating+peering, 4 unknown, 416 active+clean; 584 KiB data, 405 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.3 KiB/s wr, 2 op/s 2024-03-07T03:24:07.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:06 smithi148 bash[79819]: audit 2024-03-07T03:24:06.060073+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.148:0/2929061039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:07.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:06 smithi148 bash[82222]: audit 2024-03-07T03:24:06.060073+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.148:0/2929061039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:07.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:06 smithi195 bash[66542]: audit 2024-03-07T03:24:06.060073+0000 mon.c (mon.2) 52 : audit 0 from='client.? 172.21.15.148:0/2929061039' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:07.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:24:07] "GET /metrics HTTP/1.1" 200 55922 "" "Prometheus/2.43.0" 2024-03-07T03:24:08.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:07 smithi148 bash[82222]: cluster 2024-03-07T03:24:06.373922+0000 mgr.y (mgr.34101) 293 : cluster 0 pgmap v567: 425 pgs: 425 active+clean; 584 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-03-07T03:24:08.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:07 smithi148 bash[82222]: cluster 2024-03-07T03:24:07.235020+0000 mon.a (mon.0) 939 : cluster 0 osdmap e644: 8 total, 8 up, 8 in 2024-03-07T03:24:08.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:07 smithi148 bash[79819]: cluster 2024-03-07T03:24:06.373922+0000 mgr.y (mgr.34101) 293 : cluster 0 pgmap v567: 425 pgs: 425 active+clean; 584 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-03-07T03:24:08.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:07 smithi148 bash[79819]: cluster 2024-03-07T03:24:07.235020+0000 mon.a (mon.0) 939 : cluster 0 osdmap e644: 8 total, 8 up, 8 in 2024-03-07T03:24:08.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:07 smithi195 bash[66542]: cluster 2024-03-07T03:24:06.373922+0000 mgr.y (mgr.34101) 293 : cluster 0 pgmap v567: 425 pgs: 425 active+clean; 584 KiB data, 411 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 1.9 KiB/s wr, 5 op/s 2024-03-07T03:24:08.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:07 smithi195 bash[66542]: cluster 2024-03-07T03:24:07.235020+0000 mon.a (mon.0) 939 : cluster 0 osdmap e644: 8 total, 8 up, 8 in 2024-03-07T03:24:09.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:08 smithi148 bash[79819]: audit 2024-03-07T03:24:08.422632+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.148:0/2783697640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:09.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:08 smithi148 bash[82222]: audit 2024-03-07T03:24:08.422632+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.148:0/2783697640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:09.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:08 smithi195 bash[66542]: audit 2024-03-07T03:24:08.422632+0000 mon.c (mon.2) 53 : audit 0 from='client.? 172.21.15.148:0/2783697640' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:10.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:09 smithi148 bash[79819]: cluster 2024-03-07T03:24:08.376641+0000 mgr.y (mgr.34101) 294 : cluster 0 pgmap v569: 425 pgs: 425 active+clean; 4.6 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 684 KiB/s wr, 10 op/s 2024-03-07T03:24:10.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:09 smithi148 bash[82222]: cluster 2024-03-07T03:24:08.376641+0000 mgr.y (mgr.34101) 294 : cluster 0 pgmap v569: 425 pgs: 425 active+clean; 4.6 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 684 KiB/s wr, 10 op/s 2024-03-07T03:24:10.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:09 smithi195 bash[66542]: cluster 2024-03-07T03:24:08.376641+0000 mgr.y (mgr.34101) 294 : cluster 0 pgmap v569: 425 pgs: 425 active+clean; 4.6 MiB data, 437 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 684 KiB/s wr, 10 op/s 2024-03-07T03:24:12.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:11 smithi148 bash[79819]: cluster 2024-03-07T03:24:10.378326+0000 mgr.y (mgr.34101) 295 : cluster 0 pgmap v570: 425 pgs: 425 active+clean; 4.6 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 575 KiB/s wr, 8 op/s 2024-03-07T03:24:12.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:11 smithi148 bash[79819]: audit 2024-03-07T03:24:10.782346+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.148:0/1201493129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:12.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:11 smithi148 bash[82222]: cluster 2024-03-07T03:24:10.378326+0000 mgr.y (mgr.34101) 295 : cluster 0 pgmap v570: 425 pgs: 425 active+clean; 4.6 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 575 KiB/s wr, 8 op/s 2024-03-07T03:24:12.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:11 smithi148 bash[82222]: audit 2024-03-07T03:24:10.782346+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.148:0/1201493129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:12.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:11 smithi195 bash[66542]: cluster 2024-03-07T03:24:10.378326+0000 mgr.y (mgr.34101) 295 : cluster 0 pgmap v570: 425 pgs: 425 active+clean; 4.6 MiB data, 438 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 575 KiB/s wr, 8 op/s 2024-03-07T03:24:12.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:11 smithi195 bash[66542]: audit 2024-03-07T03:24:10.782346+0000 mon.c (mon.2) 54 : audit 0 from='client.? 172.21.15.148:0/1201493129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:12 smithi195 bash[59134]: ts=2024-03-07T03:24:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.323318+0000 mon.a (mon.0) 940 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.326193+0000 mon.a (mon.0) 941 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: cluster 2024-03-07T03:24:12.407935+0000 mgr.y (mgr.34101) 296 : cluster 0 pgmap v571: 425 pgs: 425 active+clean; 4.6 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 502 KiB/s wr, 7 op/s 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.471916+0000 mon.a (mon.0) 942 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.3", "id": [7, 5]}]: dispatch 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.472347+0000 mon.a (mon.0) 943 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 4]}]: dispatch 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.472674+0000 mon.a (mon.0) 944 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1a", "id": [7, 4]}]: dispatch 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.472994+0000 mon.a (mon.0) 945 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1d", "id": [7, 5]}]: dispatch 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.473350+0000 mon.a (mon.0) 946 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1f", "id": [7, 0]}]: dispatch 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.473677+0000 mon.a (mon.0) 947 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]: dispatch 2024-03-07T03:24:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.473998+0000 mon.a (mon.0) 948 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.474357+0000 mon.a (mon.0) 949 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.474685+0000 mon.a (mon.0) 950 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [6, 5]}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.475031+0000 mon.a (mon.0) 951 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.10", "id": [6, 5]}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:12.638194+0000 mon.a (mon.0) 952 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[79819]: audit 2024-03-07T03:24:13.133139+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.148:0/2085980898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.323318+0000 mon.a (mon.0) 940 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.326193+0000 mon.a (mon.0) 941 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: cluster 2024-03-07T03:24:12.407935+0000 mgr.y (mgr.34101) 296 : cluster 0 pgmap v571: 425 pgs: 425 active+clean; 4.6 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 502 KiB/s wr, 7 op/s 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.471916+0000 mon.a (mon.0) 942 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.3", "id": [7, 5]}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.472347+0000 mon.a (mon.0) 943 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 4]}]: dispatch 2024-03-07T03:24:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.472674+0000 mon.a (mon.0) 944 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1a", "id": [7, 4]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.472994+0000 mon.a (mon.0) 945 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1d", "id": [7, 5]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.473350+0000 mon.a (mon.0) 946 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1f", "id": [7, 0]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.473677+0000 mon.a (mon.0) 947 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.473998+0000 mon.a (mon.0) 948 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.474357+0000 mon.a (mon.0) 949 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.474685+0000 mon.a (mon.0) 950 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [6, 5]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.475031+0000 mon.a (mon.0) 951 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.10", "id": [6, 5]}]: dispatch 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:12.638194+0000 mon.a (mon.0) 952 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:13.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:13 smithi148 bash[82222]: audit 2024-03-07T03:24:13.133139+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.148:0/2085980898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:13.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.323318+0000 mon.a (mon.0) 940 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.326193+0000 mon.a (mon.0) 941 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: cluster 2024-03-07T03:24:12.407935+0000 mgr.y (mgr.34101) 296 : cluster 0 pgmap v571: 425 pgs: 425 active+clean; 4.6 MiB data, 440 MiB used, 715 GiB / 715 GiB avail; 4.2 KiB/s rd, 502 KiB/s wr, 7 op/s 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.471916+0000 mon.a (mon.0) 942 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.3", "id": [7, 5]}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.472347+0000 mon.a (mon.0) 943 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 4]}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.472674+0000 mon.a (mon.0) 944 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1a", "id": [7, 4]}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.472994+0000 mon.a (mon.0) 945 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1d", "id": [7, 5]}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.473350+0000 mon.a (mon.0) 946 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1f", "id": [7, 0]}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.473677+0000 mon.a (mon.0) 947 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.473998+0000 mon.a (mon.0) 948 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]: dispatch 2024-03-07T03:24:13.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.474357+0000 mon.a (mon.0) 949 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]: dispatch 2024-03-07T03:24:13.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.474685+0000 mon.a (mon.0) 950 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [6, 5]}]: dispatch 2024-03-07T03:24:13.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.475031+0000 mon.a (mon.0) 951 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.10", "id": [6, 5]}]: dispatch 2024-03-07T03:24:13.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:12.638194+0000 mon.a (mon.0) 952 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:13.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:13 smithi195 bash[66542]: audit 2024-03-07T03:24:13.133139+0000 mon.c (mon.2) 55 : audit 0 from='client.? 172.21.15.148:0/2085980898' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:14.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.325758+0000 mon.a (mon.0) 953 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.3", "id": [7, 5]}]': finished 2024-03-07T03:24:14.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.325993+0000 mon.a (mon.0) 954 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 4]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.326158+0000 mon.a (mon.0) 955 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1a", "id": [7, 4]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.326312+0000 mon.a (mon.0) 956 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1d", "id": [7, 5]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.326481+0000 mon.a (mon.0) 957 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1f", "id": [7, 0]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.326651+0000 mon.a (mon.0) 958 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.326787+0000 mon.a (mon.0) 959 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.326955+0000 mon.a (mon.0) 960 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.327173+0000 mon.a (mon.0) 961 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [6, 5]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.327347+0000 mon.a (mon.0) 962 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.10", "id": [6, 5]}]': finished 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: cluster 2024-03-07T03:24:13.339586+0000 mon.a (mon.0) 963 : cluster 0 osdmap e645: 8 total, 8 up, 8 in 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[82222]: audit 2024-03-07T03:24:13.809066+0000 mon.a (mon.0) 964 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:14.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.325758+0000 mon.a (mon.0) 953 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.3", "id": [7, 5]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.325993+0000 mon.a (mon.0) 954 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 4]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.326158+0000 mon.a (mon.0) 955 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1a", "id": [7, 4]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.326312+0000 mon.a (mon.0) 956 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1d", "id": [7, 5]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.326481+0000 mon.a (mon.0) 957 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1f", "id": [7, 0]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.326651+0000 mon.a (mon.0) 958 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.326787+0000 mon.a (mon.0) 959 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.326955+0000 mon.a (mon.0) 960 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.327173+0000 mon.a (mon.0) 961 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [6, 5]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.327347+0000 mon.a (mon.0) 962 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.10", "id": [6, 5]}]': finished 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: cluster 2024-03-07T03:24:13.339586+0000 mon.a (mon.0) 963 : cluster 0 osdmap e645: 8 total, 8 up, 8 in 2024-03-07T03:24:14.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:14 smithi148 bash[79819]: audit 2024-03-07T03:24:13.809066+0000 mon.a (mon.0) 964 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.325758+0000 mon.a (mon.0) 953 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.3", "id": [7, 5]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.325993+0000 mon.a (mon.0) 954 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 4]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.326158+0000 mon.a (mon.0) 955 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1a", "id": [7, 4]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.326312+0000 mon.a (mon.0) 956 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.1d", "id": [7, 5]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.326481+0000 mon.a (mon.0) 957 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.1f", "id": [7, 0]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.326651+0000 mon.a (mon.0) 958 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.9", "id": [0, 4]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.326787+0000 mon.a (mon.0) 959 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.13", "id": [0, 2]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.326955+0000 mon.a (mon.0) 960 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.4", "id": [6, 4]}]': finished 2024-03-07T03:24:14.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.327173+0000 mon.a (mon.0) 961 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.10", "id": [6, 5]}]': finished 2024-03-07T03:24:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.327347+0000 mon.a (mon.0) 962 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.10", "id": [6, 5]}]': finished 2024-03-07T03:24:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: cluster 2024-03-07T03:24:13.339586+0000 mon.a (mon.0) 963 : cluster 0 osdmap e645: 8 total, 8 up, 8 in 2024-03-07T03:24:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:14 smithi195 bash[66542]: audit 2024-03-07T03:24:13.809066+0000 mon.a (mon.0) 964 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:15.607 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[82222]: audit 2024-03-07T03:24:13.809509+0000 mgr.y (mgr.34101) 297 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:15.607 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[82222]: cephadm 2024-03-07T03:24:13.813294+0000 mgr.y (mgr.34101) 298 : cephadm 1 Upgrade: osd.3 is safe to restart 2024-03-07T03:24:15.607 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[82222]: cluster 2024-03-07T03:24:14.333554+0000 mon.a (mon.0) 965 : cluster 0 osdmap e646: 8 total, 8 up, 8 in 2024-03-07T03:24:15.607 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[82222]: cluster 2024-03-07T03:24:14.408690+0000 mgr.y (mgr.34101) 299 : cluster 0 pgmap v574: 425 pgs: 425 active+clean; 9.6 MiB data, 449 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-03-07T03:24:15.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[79819]: audit 2024-03-07T03:24:13.809509+0000 mgr.y (mgr.34101) 297 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:15.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[79819]: cephadm 2024-03-07T03:24:13.813294+0000 mgr.y (mgr.34101) 298 : cephadm 1 Upgrade: osd.3 is safe to restart 2024-03-07T03:24:15.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[79819]: cluster 2024-03-07T03:24:14.333554+0000 mon.a (mon.0) 965 : cluster 0 osdmap e646: 8 total, 8 up, 8 in 2024-03-07T03:24:15.608 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:15 smithi148 bash[79819]: cluster 2024-03-07T03:24:14.408690+0000 mgr.y (mgr.34101) 299 : cluster 0 pgmap v574: 425 pgs: 425 active+clean; 9.6 MiB data, 449 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-03-07T03:24:15.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:15 smithi195 bash[66542]: audit 2024-03-07T03:24:13.809509+0000 mgr.y (mgr.34101) 297 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-07T03:24:15.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:15 smithi195 bash[66542]: cephadm 2024-03-07T03:24:13.813294+0000 mgr.y (mgr.34101) 298 : cephadm 1 Upgrade: osd.3 is safe to restart 2024-03-07T03:24:15.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:15 smithi195 bash[66542]: cluster 2024-03-07T03:24:14.333554+0000 mon.a (mon.0) 965 : cluster 0 osdmap e646: 8 total, 8 up, 8 in 2024-03-07T03:24:15.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:15 smithi195 bash[66542]: cluster 2024-03-07T03:24:14.408690+0000 mgr.y (mgr.34101) 299 : cluster 0 pgmap v574: 425 pgs: 425 active+clean; 9.6 MiB data, 449 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.3 MiB/s wr, 6 op/s 2024-03-07T03:24:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:16 smithi148 bash[79819]: audit 2024-03-07T03:24:15.493837+0000 mon.a (mon.0) 966 : audit 0 from='client.? 172.21.15.148:0/1727195631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:16.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:16 smithi148 bash[82222]: audit 2024-03-07T03:24:15.493837+0000 mon.a (mon.0) 966 : audit 0 from='client.? 172.21.15.148:0/1727195631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:16.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:16 smithi195 bash[66542]: audit 2024-03-07T03:24:15.493837+0000 mon.a (mon.0) 966 : audit 0 from='client.? 172.21.15.148:0/1727195631' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:17.425 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:24:17] "GET /metrics HTTP/1.1" 200 60713 "" "Prometheus/2.43.0" 2024-03-07T03:24:17.702 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[79819]: cluster 2024-03-07T03:24:16.409933+0000 mgr.y (mgr.34101) 300 : cluster 0 pgmap v575: 425 pgs: 425 active+clean; 9.6 MiB data, 457 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 638 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-03-07T03:24:17.702 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[79819]: cephadm 2024-03-07T03:24:16.434257+0000 mgr.y (mgr.34101) 301 : cephadm 1 Upgrade: Updating osd.3 2024-03-07T03:24:17.703 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[79819]: audit 2024-03-07T03:24:16.440510+0000 mon.a (mon.0) 967 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:17.703 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[79819]: audit 2024-03-07T03:24:16.443169+0000 mon.a (mon.0) 968 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:24:17.703 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[79819]: audit 2024-03-07T03:24:16.444519+0000 mon.a (mon.0) 969 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:17.703 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[79819]: cephadm 2024-03-07T03:24:16.452225+0000 mgr.y (mgr.34101) 302 : cephadm 1 Deploying daemon osd.3 on smithi148 2024-03-07T03:24:17.703 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[82222]: cluster 2024-03-07T03:24:16.409933+0000 mgr.y (mgr.34101) 300 : cluster 0 pgmap v575: 425 pgs: 425 active+clean; 9.6 MiB data, 457 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 638 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-03-07T03:24:17.704 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[82222]: cephadm 2024-03-07T03:24:16.434257+0000 mgr.y (mgr.34101) 301 : cephadm 1 Upgrade: Updating osd.3 2024-03-07T03:24:17.704 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[82222]: audit 2024-03-07T03:24:16.440510+0000 mon.a (mon.0) 967 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:17.704 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[82222]: audit 2024-03-07T03:24:16.443169+0000 mon.a (mon.0) 968 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:24:17.704 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[82222]: audit 2024-03-07T03:24:16.444519+0000 mon.a (mon.0) 969 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:17.704 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:17 smithi148 bash[82222]: cephadm 2024-03-07T03:24:16.452225+0000 mgr.y (mgr.34101) 302 : cephadm 1 Deploying daemon osd.3 on smithi148 2024-03-07T03:24:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:17 smithi195 bash[66542]: cluster 2024-03-07T03:24:16.409933+0000 mgr.y (mgr.34101) 300 : cluster 0 pgmap v575: 425 pgs: 425 active+clean; 9.6 MiB data, 457 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 638 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2024-03-07T03:24:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:17 smithi195 bash[66542]: cephadm 2024-03-07T03:24:16.434257+0000 mgr.y (mgr.34101) 301 : cephadm 1 Upgrade: Updating osd.3 2024-03-07T03:24:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:17 smithi195 bash[66542]: audit 2024-03-07T03:24:16.440510+0000 mon.a (mon.0) 967 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:17 smithi195 bash[66542]: audit 2024-03-07T03:24:16.443169+0000 mon.a (mon.0) 968 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-07T03:24:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:17 smithi195 bash[66542]: audit 2024-03-07T03:24:16.444519+0000 mon.a (mon.0) 969 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:17 smithi195 bash[66542]: cephadm 2024-03-07T03:24:16.452225+0000 mgr.y (mgr.34101) 302 : cephadm 1 Deploying daemon osd.3 on smithi148 2024-03-07T03:24:18.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:18 smithi148 bash[79819]: audit 2024-03-07T03:24:17.857587+0000 mon.a (mon.0) 970 : audit 0 from='client.? 172.21.15.148:0/3201102120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:18.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:18 smithi148 bash[82222]: audit 2024-03-07T03:24:17.857587+0000 mon.a (mon.0) 970 : audit 0 from='client.? 172.21.15.148:0/3201102120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:18.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:18 smithi195 bash[66542]: audit 2024-03-07T03:24:17.857587+0000 mon.a (mon.0) 970 : audit 0 from='client.? 172.21.15.148:0/3201102120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:19.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:19 smithi148 bash[79819]: cluster 2024-03-07T03:24:18.412685+0000 mgr.y (mgr.34101) 303 : cluster 0 pgmap v576: 425 pgs: 425 active+clean; 15 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:19.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:19 smithi148 bash[82222]: cluster 2024-03-07T03:24:18.412685+0000 mgr.y (mgr.34101) 303 : cluster 0 pgmap v576: 425 pgs: 425 active+clean; 15 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:19.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:19 smithi195 bash[66542]: cluster 2024-03-07T03:24:18.412685+0000 mgr.y (mgr.34101) 303 : cluster 0 pgmap v576: 425 pgs: 425 active+clean; 15 MiB data, 468 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.2 MiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:20.696 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:20 smithi148 bash[82222]: audit 2024-03-07T03:24:20.199115+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.148:0/2810326369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:20.697 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:20 smithi148 bash[79819]: audit 2024-03-07T03:24:20.199115+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.148:0/2810326369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:20.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:20 smithi195 bash[66542]: audit 2024-03-07T03:24:20.199115+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.148:0/2810326369' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:20.947 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.947 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.948 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.948 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.948 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.948 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.949 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.949 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.949 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:20.949 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:20 smithi148 systemd[1]: Stopping Ceph osd.3 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:24:21.449 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[51019]: debug 2024-03-07T03:24:21.056+0000 7fa0e863f700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:24:21.449 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[51019]: debug 2024-03-07T03:24:21.056+0000 7fa0e863f700 -1 osd.3 646 *** Got signal Terminated *** 2024-03-07T03:24:21.449 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[51019]: debug 2024-03-07T03:24:21.056+0000 7fa0e863f700 -1 osd.3 646 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:24:21.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[79819]: debug 2024-03-07T03:24:21.456+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:21.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[79819]: cluster 2024-03-07T03:24:20.414368+0000 mgr.y (mgr.34101) 304 : cluster 0 pgmap v577: 425 pgs: 425 active+clean; 15 MiB data, 475 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1.2 MiB/s wr, 2 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:21.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[79819]: cluster 2024-03-07T03:24:21.063217+0000 mon.a (mon.0) 972 : cluster 1 osd.3 marked itself down and dead 2024-03-07T03:24:21.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[82222]: cluster 2024-03-07T03:24:20.414368+0000 mgr.y (mgr.34101) 304 : cluster 0 pgmap v577: 425 pgs: 425 active+clean; 15 MiB data, 475 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1.2 MiB/s wr, 2 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:21.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:21 smithi148 bash[82222]: cluster 2024-03-07T03:24:21.063217+0000 mon.a (mon.0) 972 : cluster 1 osd.3 marked itself down and dead 2024-03-07T03:24:21.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:21 smithi195 bash[66542]: cluster 2024-03-07T03:24:20.414368+0000 mgr.y (mgr.34101) 304 : cluster 0 pgmap v577: 425 pgs: 425 active+clean; 15 MiB data, 475 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 1.2 MiB/s wr, 2 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:21.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:21 smithi195 bash[66542]: cluster 2024-03-07T03:24:21.063217+0000 mon.a (mon.0) 972 : cluster 1 osd.3 marked itself down and dead 2024-03-07T03:24:22.386 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (24937 ms) 2024-03-07T03:24:22.386 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestIO 2024-03-07T03:24:22.386 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:24:22.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:22 smithi195 bash[59134]: ts=2024-03-07T03:24:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:22 smithi195 bash[66542]: cluster 2024-03-07T03:24:21.462597+0000 mon.a (mon.0) 973 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:24:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:22 smithi195 bash[66542]: cluster 2024-03-07T03:24:21.462655+0000 mon.a (mon.0) 974 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:22 smithi195 bash[66542]: cluster 2024-03-07T03:24:21.479290+0000 mon.a (mon.0) 975 : cluster 0 osdmap e647: 8 total, 7 up, 8 in 2024-03-07T03:24:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:22 smithi195 bash[66542]: cluster 2024-03-07T03:24:22.229472+0000 mon.a (mon.0) 976 : cluster 3 pool 'test-librbd-smithi148-68562-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:24:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:22 smithi195 bash[66542]: cluster 2024-03-07T03:24:22.230058+0000 mon.a (mon.0) 977 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:24:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:22 smithi195 bash[66542]: cluster 2024-03-07T03:24:22.236176+0000 mon.a (mon.0) 978 : cluster 0 osdmap e648: 8 total, 7 up, 8 in 2024-03-07T03:24:22.778 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[79819]: cluster 2024-03-07T03:24:21.462597+0000 mon.a (mon.0) 973 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:24:22.779 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[79819]: cluster 2024-03-07T03:24:21.462655+0000 mon.a (mon.0) 974 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:22.779 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[79819]: cluster 2024-03-07T03:24:21.479290+0000 mon.a (mon.0) 975 : cluster 0 osdmap e647: 8 total, 7 up, 8 in 2024-03-07T03:24:22.779 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[79819]: cluster 2024-03-07T03:24:22.229472+0000 mon.a (mon.0) 976 : cluster 3 pool 'test-librbd-smithi148-68562-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:24:22.779 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[79819]: cluster 2024-03-07T03:24:22.230058+0000 mon.a (mon.0) 977 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:24:22.779 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[79819]: cluster 2024-03-07T03:24:22.236176+0000 mon.a (mon.0) 978 : cluster 0 osdmap e648: 8 total, 7 up, 8 in 2024-03-07T03:24:22.779 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[82222]: cluster 2024-03-07T03:24:21.462597+0000 mon.a (mon.0) 973 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:24:22.780 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[82222]: cluster 2024-03-07T03:24:21.462655+0000 mon.a (mon.0) 974 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:22.780 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[82222]: cluster 2024-03-07T03:24:21.479290+0000 mon.a (mon.0) 975 : cluster 0 osdmap e647: 8 total, 7 up, 8 in 2024-03-07T03:24:22.780 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[82222]: cluster 2024-03-07T03:24:22.229472+0000 mon.a (mon.0) 976 : cluster 3 pool 'test-librbd-smithi148-68562-25' is full (reached quota's max_bytes: 10 MiB) 2024-03-07T03:24:22.780 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[82222]: cluster 2024-03-07T03:24:22.230058+0000 mon.a (mon.0) 977 : cluster 3 Health check failed: 1 pool(s) full (POOL_FULL) 2024-03-07T03:24:22.780 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[82222]: cluster 2024-03-07T03:24:22.236176+0000 mon.a (mon.0) 978 : cluster 0 osdmap e648: 8 total, 7 up, 8 in 2024-03-07T03:24:23.036 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:22 smithi148 bash[103782]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:24:23.036 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:23 smithi148 bash[103874]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:24:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:23 smithi148 bash[82222]: cluster 2024-03-07T03:24:22.414889+0000 mgr.y (mgr.34101) 305 : cluster 0 pgmap v580: 425 pgs: 60 stale+active+clean, 365 active+clean; 16 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 768 KiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:23.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:23 smithi148 bash[82222]: audit 2024-03-07T03:24:22.634927+0000 mon.a (mon.0) 979 : audit 0 from='client.? 172.21.15.148:0/2799416467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:23 smithi148 bash[79819]: cluster 2024-03-07T03:24:22.414889+0000 mgr.y (mgr.34101) 305 : cluster 0 pgmap v580: 425 pgs: 60 stale+active+clean, 365 active+clean; 16 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 768 KiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:23 smithi148 bash[79819]: audit 2024-03-07T03:24:22.634927+0000 mon.a (mon.0) 979 : audit 0 from='client.? 172.21.15.148:0/2799416467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:23.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:23 smithi195 bash[66542]: cluster 2024-03-07T03:24:22.414889+0000 mgr.y (mgr.34101) 305 : cluster 0 pgmap v580: 425 pgs: 60 stale+active+clean, 365 active+clean; 16 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 768 KiB/s wr, 3 op/s; 2 B/s, 0 objects/s recovering 2024-03-07T03:24:23.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:23 smithi195 bash[66542]: audit 2024-03-07T03:24:22.634927+0000 mon.a (mon.0) 979 : audit 0 from='client.? 172.21.15.148:0/2799416467' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:24.710 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:24.710 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:24.711 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:24.712 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:24.713 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:24.714 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.715 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:24.716 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 512~512 = 512 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 1536~512 = 0 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.717 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.718 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.719 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.720 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.721 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.722 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:24.723 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.724 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.725 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:24.726 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:24.765 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:24 smithi148 bash[79819]: cluster 2024-03-07T03:24:24.470432+0000 mon.a (mon.0) 980 : cluster 3 Health check failed: Degraded data redundancy: 43/519 objects degraded (8.285%), 28 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:24.766 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:24 smithi148 bash[82222]: cluster 2024-03-07T03:24:24.470432+0000 mon.a (mon.0) 980 : cluster 3 Health check failed: Degraded data redundancy: 43/519 objects degraded (8.285%), 28 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:24.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:24 smithi195 bash[66542]: cluster 2024-03-07T03:24:24.470432+0000 mon.a (mon.0) 980 : cluster 3 Health check failed: Degraded data redundancy: 43/519 objects degraded (8.285%), 28 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:25.789 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.789 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.790 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.790 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-07T03:24:25.790 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: Stopped Ceph osd.3 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:24:25.790 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:25 smithi148 bash[79819]: cluster 2024-03-07T03:24:24.416393+0000 mgr.y (mgr.34101) 306 : cluster 0 pgmap v581: 425 pgs: 28 active+undersized+degraded, 71 active+undersized, 27 stale+active+clean, 299 active+clean; 1.6 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.0 MiB/s wr, 11 op/s; 43/519 objects degraded (8.285%); 2 B/s, 0 objects/s recovering 2024-03-07T03:24:25.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:25 smithi148 bash[79819]: audit 2024-03-07T03:24:25.025837+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.148:0/3995638354' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:25.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:25 smithi148 bash[82222]: cluster 2024-03-07T03:24:24.416393+0000 mgr.y (mgr.34101) 306 : cluster 0 pgmap v581: 425 pgs: 28 active+undersized+degraded, 71 active+undersized, 27 stale+active+clean, 299 active+clean; 1.6 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.0 MiB/s wr, 11 op/s; 43/519 objects degraded (8.285%); 2 B/s, 0 objects/s recovering 2024-03-07T03:24:25.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:25 smithi148 bash[82222]: audit 2024-03-07T03:24:25.025837+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.148:0/3995638354' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:25.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.791 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.792 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.792 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:25 smithi195 bash[66542]: cluster 2024-03-07T03:24:24.416393+0000 mgr.y (mgr.34101) 306 : cluster 0 pgmap v581: 425 pgs: 28 active+undersized+degraded, 71 active+undersized, 27 stale+active+clean, 299 active+clean; 1.6 MiB data, 456 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 1.0 MiB/s wr, 11 op/s; 43/519 objects degraded (8.285%); 2 B/s, 0 objects/s recovering 2024-03-07T03:24:25.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:25 smithi195 bash[66542]: audit 2024-03-07T03:24:25.025837+0000 mon.a (mon.0) 981 : audit 0 from='client.? 172.21.15.148:0/3995638354' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:26.289 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:25 smithi148 systemd[1]: Started Ceph osd.3 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:24:26.770 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.771 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.772 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.773 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.774 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.775 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.776 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.777 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.778 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestIO (4385 ms) 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-03-07T03:24:26.779 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:24:27.165 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[79819]: audit 2024-03-07T03:24:25.877576+0000 mon.a (mon.0) 982 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:27.165 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[79819]: audit 2024-03-07T03:24:25.884219+0000 mon.a (mon.0) 983 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:27.165 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[79819]: audit 2024-03-07T03:24:25.933227+0000 mon.a (mon.0) 984 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:27.165 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[79819]: cluster 2024-03-07T03:24:26.417820+0000 mgr.y (mgr.34101) 307 : cluster 0 pgmap v582: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 455 KiB/s rd, 622 KiB/s wr, 921 op/s; 74/516 objects degraded (14.341%) 2024-03-07T03:24:27.166 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[82222]: audit 2024-03-07T03:24:25.877576+0000 mon.a (mon.0) 982 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:27.166 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[82222]: audit 2024-03-07T03:24:25.884219+0000 mon.a (mon.0) 983 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:27.166 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[82222]: audit 2024-03-07T03:24:25.933227+0000 mon.a (mon.0) 984 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:27.166 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:26 smithi148 bash[82222]: cluster 2024-03-07T03:24:26.417820+0000 mgr.y (mgr.34101) 307 : cluster 0 pgmap v582: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 455 KiB/s rd, 622 KiB/s wr, 921 op/s; 74/516 objects degraded (14.341%) 2024-03-07T03:24:27.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:26 smithi195 bash[66542]: audit 2024-03-07T03:24:25.877576+0000 mon.a (mon.0) 982 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:27.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:26 smithi195 bash[66542]: audit 2024-03-07T03:24:25.884219+0000 mon.a (mon.0) 983 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:27.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:26 smithi195 bash[66542]: audit 2024-03-07T03:24:25.933227+0000 mon.a (mon.0) 984 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:27.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:26 smithi195 bash[66542]: cluster 2024-03-07T03:24:26.417820+0000 mgr.y (mgr.34101) 307 : cluster 0 pgmap v582: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 455 KiB/s rd, 622 KiB/s wr, 921 op/s; 74/516 objects degraded (14.341%) 2024-03-07T03:24:27.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:24:27] "GET /metrics HTTP/1.1" 200 60713 "" "Prometheus/2.43.0" 2024-03-07T03:24:27.874 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[104047]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-07T03:24:27.874 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[104047]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-07T03:24:27.874 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[104047]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-07T03:24:27.874 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[104047]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-07T03:24:27.874 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[104047]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-07T03:24:27.874 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[104047]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-07T03:24:27.874 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[104047]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-07T03:24:28.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[79819]: cluster 2024-03-07T03:24:27.231572+0000 mon.a (mon.0) 985 : cluster 1 pool 'test-librbd-smithi148-68562-25' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:24:28.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[79819]: cluster 2024-03-07T03:24:27.231850+0000 mon.a (mon.0) 986 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:24:28.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[79819]: cluster 2024-03-07T03:24:27.241419+0000 mon.a (mon.0) 987 : cluster 0 osdmap e649: 8 total, 7 up, 8 in 2024-03-07T03:24:28.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[79819]: audit 2024-03-07T03:24:27.318563+0000 mon.a (mon.0) 988 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:28.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[79819]: audit 2024-03-07T03:24:27.320087+0000 mon.a (mon.0) 989 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:28.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[79819]: audit 2024-03-07T03:24:27.389790+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.148:0/4279227627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:28.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[82222]: cluster 2024-03-07T03:24:27.231572+0000 mon.a (mon.0) 985 : cluster 1 pool 'test-librbd-smithi148-68562-25' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:24:28.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[82222]: cluster 2024-03-07T03:24:27.231850+0000 mon.a (mon.0) 986 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:24:28.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[82222]: cluster 2024-03-07T03:24:27.241419+0000 mon.a (mon.0) 987 : cluster 0 osdmap e649: 8 total, 7 up, 8 in 2024-03-07T03:24:28.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[82222]: audit 2024-03-07T03:24:27.318563+0000 mon.a (mon.0) 988 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:28.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[82222]: audit 2024-03-07T03:24:27.320087+0000 mon.a (mon.0) 989 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:28.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:27 smithi148 bash[82222]: audit 2024-03-07T03:24:27.389790+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.148:0/4279227627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:28.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:27 smithi195 bash[66542]: cluster 2024-03-07T03:24:27.231572+0000 mon.a (mon.0) 985 : cluster 1 pool 'test-librbd-smithi148-68562-25' no longer out of quota; removing NO_QUOTA flag 2024-03-07T03:24:28.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:27 smithi195 bash[66542]: cluster 2024-03-07T03:24:27.231850+0000 mon.a (mon.0) 986 : cluster 1 Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-03-07T03:24:28.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:27 smithi195 bash[66542]: cluster 2024-03-07T03:24:27.241419+0000 mon.a (mon.0) 987 : cluster 0 osdmap e649: 8 total, 7 up, 8 in 2024-03-07T03:24:28.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:27 smithi195 bash[66542]: audit 2024-03-07T03:24:27.318563+0000 mon.a (mon.0) 988 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:28.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:27 smithi195 bash[66542]: audit 2024-03-07T03:24:27.320087+0000 mon.a (mon.0) 989 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:28.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:27 smithi195 bash[66542]: audit 2024-03-07T03:24:27.389790+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.148:0/4279227627' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:29.226 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:28 smithi148 bash[79819]: cluster 2024-03-07T03:24:28.420758+0000 mgr.y (mgr.34101) 308 : cluster 0 pgmap v584: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 527 KiB/s rd, 570 KiB/s wr, 1.06k op/s; 74/522 objects degraded (14.176%) 2024-03-07T03:24:29.227 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:28 smithi148 bash[82222]: cluster 2024-03-07T03:24:28.420758+0000 mgr.y (mgr.34101) 308 : cluster 0 pgmap v584: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 527 KiB/s rd, 570 KiB/s wr, 1.06k op/s; 74/522 objects degraded (14.176%) 2024-03-07T03:24:29.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:28 smithi195 bash[66542]: cluster 2024-03-07T03:24:28.420758+0000 mgr.y (mgr.34101) 308 : cluster 0 pgmap v584: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 527 KiB/s rd, 570 KiB/s wr, 1.06k op/s; 74/522 objects degraded (14.176%) 2024-03-07T03:24:29.539 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:29 smithi148 bash[104302]: debug 2024-03-07T03:24:29.316+0000 7f62fd90f700 1 -- 172.21.15.148:0/4158421236 <== mon.0 v2:172.21.15.148:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5637f0013680 con 0x5637f004f000 2024-03-07T03:24:30.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:29 smithi148 bash[79819]: audit 2024-03-07T03:24:29.728991+0000 mon.a (mon.0) 991 : audit 0 from='client.? 172.21.15.148:0/2434547868' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:30.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:29 smithi148 bash[82222]: audit 2024-03-07T03:24:29.728991+0000 mon.a (mon.0) 991 : audit 0 from='client.? 172.21.15.148:0/2434547868' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:30.290 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:29 smithi148 bash[104302]: debug 2024-03-07T03:24:29.848+0000 7f6306aa77c0 -1 Falling back to public interface 2024-03-07T03:24:30.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:29 smithi195 bash[66542]: audit 2024-03-07T03:24:29.728991+0000 mon.a (mon.0) 991 : audit 0 from='client.? 172.21.15.148:0/2434547868' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:31.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:30 smithi148 bash[79819]: cluster 2024-03-07T03:24:30.423032+0000 mgr.y (mgr.34101) 309 : cluster 0 pgmap v585: 425 pgs: 50 active+undersized+degraded, 131 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 447 KiB/s rd, 483 KiB/s wr, 904 op/s; 75/525 objects degraded (14.286%) 2024-03-07T03:24:31.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:30 smithi148 bash[82222]: cluster 2024-03-07T03:24:30.423032+0000 mgr.y (mgr.34101) 309 : cluster 0 pgmap v585: 425 pgs: 50 active+undersized+degraded, 131 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 447 KiB/s rd, 483 KiB/s wr, 904 op/s; 75/525 objects degraded (14.286%) 2024-03-07T03:24:31.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:30 smithi195 bash[66542]: cluster 2024-03-07T03:24:30.423032+0000 mgr.y (mgr.34101) 309 : cluster 0 pgmap v585: 425 pgs: 50 active+undersized+degraded, 131 active+undersized, 244 active+clean; 590 KiB data, 438 MiB used, 715 GiB / 715 GiB avail; 447 KiB/s rd, 483 KiB/s wr, 904 op/s; 75/525 objects degraded (14.286%) 2024-03-07T03:24:32.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:31 smithi148 bash[79819]: cluster 2024-03-07T03:24:30.898104+0000 mon.a (mon.0) 992 : cluster 3 Health check update: Degraded data redundancy: 75/525 objects degraded (14.286%), 50 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:32.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:31 smithi148 bash[79819]: audit 2024-03-07T03:24:31.779255+0000 mon.a (mon.0) 993 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:32.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:31 smithi148 bash[79819]: audit 2024-03-07T03:24:31.785795+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:32.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:31 smithi148 bash[82222]: cluster 2024-03-07T03:24:30.898104+0000 mon.a (mon.0) 992 : cluster 3 Health check update: Degraded data redundancy: 75/525 objects degraded (14.286%), 50 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:32.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:31 smithi148 bash[82222]: audit 2024-03-07T03:24:31.779255+0000 mon.a (mon.0) 993 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:32.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:31 smithi148 bash[82222]: audit 2024-03-07T03:24:31.785795+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:32.303 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:31 smithi195 bash[66542]: cluster 2024-03-07T03:24:30.898104+0000 mon.a (mon.0) 992 : cluster 3 Health check update: Degraded data redundancy: 75/525 objects degraded (14.286%), 50 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:32.303 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:31 smithi195 bash[66542]: audit 2024-03-07T03:24:31.779255+0000 mon.a (mon.0) 993 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:32.303 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:31 smithi195 bash[66542]: audit 2024-03-07T03:24:31.785795+0000 mon.a (mon.0) 994 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:32.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:32 smithi195 bash[59134]: ts=2024-03-07T03:24:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:33.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[82222]: audit 2024-03-07T03:24:32.080836+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.148:0/984895222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:33.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[82222]: cluster 2024-03-07T03:24:32.423998+0000 mgr.y (mgr.34101) 310 : cluster 0 pgmap v586: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 6.6 MiB data, 441 MiB used, 715 GiB / 715 GiB avail; 408 KiB/s rd, 826 KiB/s wr, 836 op/s; 76/528 objects degraded (14.394%) 2024-03-07T03:24:33.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[82222]: audit 2024-03-07T03:24:32.776927+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:33.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[82222]: audit 2024-03-07T03:24:32.783236+0000 mon.a (mon.0) 996 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:33.290 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:33 smithi148 bash[104302]: debug 2024-03-07T03:24:33.092+0000 7f6306aa77c0 -1 osd.3 646 log_to_monitors true 2024-03-07T03:24:33.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[79819]: audit 2024-03-07T03:24:32.080836+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.148:0/984895222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:33.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[79819]: cluster 2024-03-07T03:24:32.423998+0000 mgr.y (mgr.34101) 310 : cluster 0 pgmap v586: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 6.6 MiB data, 441 MiB used, 715 GiB / 715 GiB avail; 408 KiB/s rd, 826 KiB/s wr, 836 op/s; 76/528 objects degraded (14.394%) 2024-03-07T03:24:33.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[79819]: audit 2024-03-07T03:24:32.776927+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:33.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:32 smithi148 bash[79819]: audit 2024-03-07T03:24:32.783236+0000 mon.a (mon.0) 996 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:33.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:32 smithi195 bash[66542]: audit 2024-03-07T03:24:32.080836+0000 mon.c (mon.2) 56 : audit 0 from='client.? 172.21.15.148:0/984895222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:33.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:32 smithi195 bash[66542]: cluster 2024-03-07T03:24:32.423998+0000 mgr.y (mgr.34101) 310 : cluster 0 pgmap v586: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 6.6 MiB data, 441 MiB used, 715 GiB / 715 GiB avail; 408 KiB/s rd, 826 KiB/s wr, 836 op/s; 76/528 objects degraded (14.394%) 2024-03-07T03:24:33.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:32 smithi195 bash[66542]: audit 2024-03-07T03:24:32.776927+0000 mon.a (mon.0) 995 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:33.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:32 smithi195 bash[66542]: audit 2024-03-07T03:24:32.783236+0000 mon.a (mon.0) 996 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:34.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:33 smithi148 bash[79819]: audit 2024-03-07T03:24:33.100521+0000 mon.c (mon.2) 57 : audit 1 from='osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:24:34.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:33 smithi148 bash[79819]: audit 2024-03-07T03:24:33.100818+0000 mon.a (mon.0) 997 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:24:34.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:33 smithi148 bash[82222]: audit 2024-03-07T03:24:33.100521+0000 mon.c (mon.2) 57 : audit 1 from='osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:24:34.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:33 smithi148 bash[82222]: audit 2024-03-07T03:24:33.100818+0000 mon.a (mon.0) 997 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:24:34.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:33 smithi195 bash[66542]: audit 2024-03-07T03:24:33.100521+0000 mon.c (mon.2) 57 : audit 1 from='osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:24:34.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:33 smithi195 bash[66542]: audit 2024-03-07T03:24:33.100818+0000 mon.a (mon.0) 997 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-07T03:24:34.789 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[104302]: debug 2024-03-07T03:24:34.456+0000 7f62f3ef7700 -1 osd.3 646 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.184 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:35.185 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:35.186 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.187 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:35.188 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.189 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:35.190 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 512~512 = 512 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 1536~512 = 0 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.191 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.192 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.193 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.194 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.195 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.196 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.197 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.198 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:35.199 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:35.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[79819]: audit 2024-03-07T03:24:33.934829+0000 mon.a (mon.0) 998 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:24:35.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[79819]: cluster 2024-03-07T03:24:33.942753+0000 mon.a (mon.0) 999 : cluster 0 osdmap e650: 8 total, 7 up, 8 in 2024-03-07T03:24:35.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[79819]: audit 2024-03-07T03:24:33.947186+0000 mon.c (mon.2) 58 : audit 1 from='osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:24:35.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[79819]: audit 2024-03-07T03:24:33.959394+0000 mon.a (mon.0) 1000 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:24:35.289 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[79819]: cluster 2024-03-07T03:24:34.424732+0000 mgr.y (mgr.34101) 311 : cluster 0 pgmap v588: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 6.8 MiB data, 408 MiB used, 715 GiB / 715 GiB avail; 426 KiB/s rd, 726 KiB/s wr, 503 op/s; 77/531 objects degraded (14.501%) 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[79819]: audit 2024-03-07T03:24:34.433611+0000 mon.a (mon.0) 1001 : audit 1 from='osd.3 ' entity='osd.3' 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[79819]: audit 2024-03-07T03:24:34.515587+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.148:0/2850435245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[82222]: audit 2024-03-07T03:24:33.934829+0000 mon.a (mon.0) 998 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[82222]: cluster 2024-03-07T03:24:33.942753+0000 mon.a (mon.0) 999 : cluster 0 osdmap e650: 8 total, 7 up, 8 in 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[82222]: audit 2024-03-07T03:24:33.947186+0000 mon.c (mon.2) 58 : audit 1 from='osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[82222]: audit 2024-03-07T03:24:33.959394+0000 mon.a (mon.0) 1000 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[82222]: cluster 2024-03-07T03:24:34.424732+0000 mgr.y (mgr.34101) 311 : cluster 0 pgmap v588: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 6.8 MiB data, 408 MiB used, 715 GiB / 715 GiB avail; 426 KiB/s rd, 726 KiB/s wr, 503 op/s; 77/531 objects degraded (14.501%) 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[82222]: audit 2024-03-07T03:24:34.433611+0000 mon.a (mon.0) 1001 : audit 1 from='osd.3 ' entity='osd.3' 2024-03-07T03:24:35.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:34 smithi148 bash[82222]: audit 2024-03-07T03:24:34.515587+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.148:0/2850435245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:35.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:34 smithi195 bash[66542]: audit 2024-03-07T03:24:33.934829+0000 mon.a (mon.0) 998 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-07T03:24:35.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:34 smithi195 bash[66542]: cluster 2024-03-07T03:24:33.942753+0000 mon.a (mon.0) 999 : cluster 0 osdmap e650: 8 total, 7 up, 8 in 2024-03-07T03:24:35.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:34 smithi195 bash[66542]: audit 2024-03-07T03:24:33.947186+0000 mon.c (mon.2) 58 : audit 1 from='osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:24:35.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:34 smithi195 bash[66542]: audit 2024-03-07T03:24:33.959394+0000 mon.a (mon.0) 1000 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi148", "root=default"]}]: dispatch 2024-03-07T03:24:35.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:34 smithi195 bash[66542]: cluster 2024-03-07T03:24:34.424732+0000 mgr.y (mgr.34101) 311 : cluster 0 pgmap v588: 425 pgs: 49 active+undersized+degraded, 132 active+undersized, 244 active+clean; 6.8 MiB data, 408 MiB used, 715 GiB / 715 GiB avail; 426 KiB/s rd, 726 KiB/s wr, 503 op/s; 77/531 objects degraded (14.501%) 2024-03-07T03:24:35.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:34 smithi195 bash[66542]: audit 2024-03-07T03:24:34.433611+0000 mon.a (mon.0) 1001 : audit 1 from='osd.3 ' entity='osd.3' 2024-03-07T03:24:35.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:34 smithi195 bash[66542]: audit 2024-03-07T03:24:34.515587+0000 mon.c (mon.2) 59 : audit 0 from='client.? 172.21.15.148:0/2850435245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:35.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[79819]: debug 2024-03-07T03:24:35.432+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:36.289 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[82222]: cluster 2024-03-07T03:24:35.433653+0000 mon.a (mon.0) 1002 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[82222]: cluster 2024-03-07T03:24:35.433722+0000 mon.a (mon.0) 1003 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[82222]: cluster 2024-03-07T03:24:35.445515+0000 mon.a (mon.0) 1004 : cluster 1 osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435] boot 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[82222]: cluster 2024-03-07T03:24:35.445632+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e651: 8 total, 8 up, 8 in 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[82222]: audit 2024-03-07T03:24:35.447266+0000 mon.a (mon.0) 1006 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[79819]: cluster 2024-03-07T03:24:35.433653+0000 mon.a (mon.0) 1002 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[79819]: cluster 2024-03-07T03:24:35.433722+0000 mon.a (mon.0) 1003 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[79819]: cluster 2024-03-07T03:24:35.445515+0000 mon.a (mon.0) 1004 : cluster 1 osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435] boot 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[79819]: cluster 2024-03-07T03:24:35.445632+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e651: 8 total, 8 up, 8 in 2024-03-07T03:24:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:35 smithi148 bash[79819]: audit 2024-03-07T03:24:35.447266+0000 mon.a (mon.0) 1006 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:24:36.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:35 smithi195 bash[66542]: cluster 2024-03-07T03:24:35.433653+0000 mon.a (mon.0) 1002 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:36.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:35 smithi195 bash[66542]: cluster 2024-03-07T03:24:35.433722+0000 mon.a (mon.0) 1003 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:24:36.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:35 smithi195 bash[66542]: cluster 2024-03-07T03:24:35.445515+0000 mon.a (mon.0) 1004 : cluster 1 osd.3 [v2:172.21.15.148:6826/3829359435,v1:172.21.15.148:6827/3829359435] boot 2024-03-07T03:24:36.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:35 smithi195 bash[66542]: cluster 2024-03-07T03:24:35.445632+0000 mon.a (mon.0) 1005 : cluster 0 osdmap e651: 8 total, 8 up, 8 in 2024-03-07T03:24:36.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:35 smithi195 bash[66542]: audit 2024-03-07T03:24:35.447266+0000 mon.a (mon.0) 1006 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-07T03:24:36.907 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 4 2024-03-07T03:24:37.429 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:24:37] "GET /metrics HTTP/1.1" 200 60713 "" "Prometheus/2.43.0" 2024-03-07T03:24:37.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[79819]: cluster 2024-03-07T03:24:36.426661+0000 mgr.y (mgr.34101) 312 : cluster 0 pgmap v590: 425 pgs: 22 peering, 42 active+undersized+degraded, 117 active+undersized, 244 active+clean; 6.8 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 423 KiB/s rd, 726 KiB/s wr, 498 op/s; 63/531 objects degraded (11.864%) 2024-03-07T03:24:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[79819]: cluster 2024-03-07T03:24:36.445014+0000 mon.a (mon.0) 1007 : cluster 0 osdmap e652: 8 total, 8 up, 8 in 2024-03-07T03:24:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[79819]: audit 2024-03-07T03:24:36.907128+0000 mon.a (mon.0) 1008 : audit 0 from='client.? 172.21.15.148:0/1893629984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:37.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[79819]: cluster 2024-03-07T03:24:36.948014+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: Degraded data redundancy: 63/531 objects degraded (11.864%), 42 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[82222]: cluster 2024-03-07T03:24:36.426661+0000 mgr.y (mgr.34101) 312 : cluster 0 pgmap v590: 425 pgs: 22 peering, 42 active+undersized+degraded, 117 active+undersized, 244 active+clean; 6.8 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 423 KiB/s rd, 726 KiB/s wr, 498 op/s; 63/531 objects degraded (11.864%) 2024-03-07T03:24:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[82222]: cluster 2024-03-07T03:24:36.445014+0000 mon.a (mon.0) 1007 : cluster 0 osdmap e652: 8 total, 8 up, 8 in 2024-03-07T03:24:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[82222]: audit 2024-03-07T03:24:36.907128+0000 mon.a (mon.0) 1008 : audit 0 from='client.? 172.21.15.148:0/1893629984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:37.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:37 smithi148 bash[82222]: cluster 2024-03-07T03:24:36.948014+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: Degraded data redundancy: 63/531 objects degraded (11.864%), 42 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:37 smithi195 bash[66542]: cluster 2024-03-07T03:24:36.426661+0000 mgr.y (mgr.34101) 312 : cluster 0 pgmap v590: 425 pgs: 22 peering, 42 active+undersized+degraded, 117 active+undersized, 244 active+clean; 6.8 MiB data, 417 MiB used, 715 GiB / 715 GiB avail; 423 KiB/s rd, 726 KiB/s wr, 498 op/s; 63/531 objects degraded (11.864%) 2024-03-07T03:24:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:37 smithi195 bash[66542]: cluster 2024-03-07T03:24:36.445014+0000 mon.a (mon.0) 1007 : cluster 0 osdmap e652: 8 total, 8 up, 8 in 2024-03-07T03:24:37.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:37 smithi195 bash[66542]: audit 2024-03-07T03:24:36.907128+0000 mon.a (mon.0) 1008 : audit 0 from='client.? 172.21.15.148:0/1893629984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-07T03:24:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:37 smithi195 bash[66542]: cluster 2024-03-07T03:24:36.948014+0000 mon.a (mon.0) 1009 : cluster 3 Health check update: Degraded data redundancy: 63/531 objects degraded (11.864%), 42 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:38.262 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch upgrade pause' 2024-03-07T03:24:39.759 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:39 smithi148 bash[79819]: cluster 2024-03-07T03:24:38.428187+0000 mgr.y (mgr.34101) 313 : cluster 0 pgmap v592: 425 pgs: 22 peering, 25 active+undersized+degraded, 68 active+undersized, 310 active+clean; 6.8 MiB data, 421 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 540 KiB/s wr, 1.04k op/s; 35/531 objects degraded (6.591%); 35 KiB/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:24:39.759 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:39 smithi148 bash[82222]: cluster 2024-03-07T03:24:38.428187+0000 mgr.y (mgr.34101) 313 : cluster 0 pgmap v592: 425 pgs: 22 peering, 25 active+undersized+degraded, 68 active+undersized, 310 active+clean; 6.8 MiB data, 421 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 540 KiB/s wr, 1.04k op/s; 35/531 objects degraded (6.591%); 35 KiB/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:24:39.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:39 smithi195 bash[66542]: cluster 2024-03-07T03:24:38.428187+0000 mgr.y (mgr.34101) 313 : cluster 0 pgmap v592: 425 pgs: 22 peering, 25 active+undersized+degraded, 68 active+undersized, 310 active+clean; 6.8 MiB data, 421 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 540 KiB/s wr, 1.04k op/s; 35/531 objects degraded (6.591%); 35 KiB/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:24:40.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:40 smithi148 bash[79819]: cluster 2024-03-07T03:24:40.447648+0000 mon.a (mon.0) 1010 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/531 objects degraded (6.591%), 25 pgs degraded) 2024-03-07T03:24:40.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:40 smithi148 bash[82222]: cluster 2024-03-07T03:24:40.447648+0000 mon.a (mon.0) 1010 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/531 objects degraded (6.591%), 25 pgs degraded) 2024-03-07T03:24:40.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:40 smithi195 bash[66542]: cluster 2024-03-07T03:24:40.447648+0000 mon.a (mon.0) 1010 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 35/531 objects degraded (6.591%), 25 pgs degraded) 2024-03-07T03:24:41.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:41 smithi148 bash[82222]: cluster 2024-03-07T03:24:40.430266+0000 mgr.y (mgr.34101) 314 : cluster 0 pgmap v593: 425 pgs: 22 peering, 403 active+clean; 6.8 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 497 KiB/s rd, 268 KiB/s wr, 499 op/s; 32 KiB/s, 6 keys/s, 0 objects/s recovering 2024-03-07T03:24:41.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:41 smithi148 bash[79819]: cluster 2024-03-07T03:24:40.430266+0000 mgr.y (mgr.34101) 314 : cluster 0 pgmap v593: 425 pgs: 22 peering, 403 active+clean; 6.8 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 497 KiB/s rd, 268 KiB/s wr, 499 op/s; 32 KiB/s, 6 keys/s, 0 objects/s recovering 2024-03-07T03:24:41.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:41 smithi195 bash[66542]: cluster 2024-03-07T03:24:40.430266+0000 mgr.y (mgr.34101) 314 : cluster 0 pgmap v593: 425 pgs: 22 peering, 403 active+clean; 6.8 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 497 KiB/s rd, 268 KiB/s wr, 499 op/s; 32 KiB/s, 6 keys/s, 0 objects/s recovering 2024-03-07T03:24:42.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:42 smithi195 bash[59134]: ts=2024-03-07T03:24:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:43.463 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:42.318606+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:42.320168+0000 mon.a (mon.0) 1012 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: cluster 2024-03-07T03:24:42.431082+0000 mgr.y (mgr.34101) 315 : cluster 0 pgmap v594: 425 pgs: 425 active+clean; 6.8 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 950 KiB/s rd, 217 KiB/s wr, 405 op/s; 538 KiB/s, 5 keys/s, 0 objects/s recovering 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.078977+0000 mon.a (mon.0) 1013 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.087824+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.089254+0000 mon.a (mon.0) 1015 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.090440+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.100395+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.614 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.178529+0000 mon.a (mon.0) 1018 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:43.615 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.181598+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.615 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.183983+0000 mon.a (mon.0) 1020 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.615 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.185894+0000 mon.a (mon.0) 1021 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.615 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:43 smithi195 bash[66542]: audit 2024-03-07T03:24:43.188061+0000 mon.a (mon.0) 1022 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:42.318606+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:42.320168+0000 mon.a (mon.0) 1012 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: cluster 2024-03-07T03:24:42.431082+0000 mgr.y (mgr.34101) 315 : cluster 0 pgmap v594: 425 pgs: 425 active+clean; 6.8 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 950 KiB/s rd, 217 KiB/s wr, 405 op/s; 538 KiB/s, 5 keys/s, 0 objects/s recovering 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.078977+0000 mon.a (mon.0) 1013 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.087824+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.089254+0000 mon.a (mon.0) 1015 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.090440+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.100395+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.178529+0000 mon.a (mon.0) 1018 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.181598+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.183983+0000 mon.a (mon.0) 1020 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.185894+0000 mon.a (mon.0) 1021 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[79819]: audit 2024-03-07T03:24:43.188061+0000 mon.a (mon.0) 1022 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:42.318606+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:42.320168+0000 mon.a (mon.0) 1012 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: cluster 2024-03-07T03:24:42.431082+0000 mgr.y (mgr.34101) 315 : cluster 0 pgmap v594: 425 pgs: 425 active+clean; 6.8 MiB data, 425 MiB used, 715 GiB / 715 GiB avail; 950 KiB/s rd, 217 KiB/s wr, 405 op/s; 538 KiB/s, 5 keys/s, 0 objects/s recovering 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.078977+0000 mon.a (mon.0) 1013 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.087824+0000 mon.a (mon.0) 1014 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.089254+0000 mon.a (mon.0) 1015 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.090440+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.100395+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.178529+0000 mon.a (mon.0) 1018 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.181598+0000 mon.a (mon.0) 1019 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.183983+0000 mon.a (mon.0) 1020 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.185894+0000 mon.a (mon.0) 1021 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:24:43.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:43 smithi148 bash[82222]: audit 2024-03-07T03:24:43.188061+0000 mon.a (mon.0) 1022 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:24:44.703 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:44 smithi195 bash[66542]: audit 2024-03-07T03:24:43.188414+0000 mgr.y (mgr.34101) 316 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:24:44.703 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:44 smithi195 bash[66542]: cephadm 2024-03-07T03:24:43.191146+0000 mgr.y (mgr.34101) 317 : cephadm 1 Upgrade: osd.4 is safe to restart 2024-03-07T03:24:44.733 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:44 smithi148 bash[79819]: audit 2024-03-07T03:24:43.188414+0000 mgr.y (mgr.34101) 316 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:24:44.733 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:44 smithi148 bash[79819]: cephadm 2024-03-07T03:24:43.191146+0000 mgr.y (mgr.34101) 317 : cephadm 1 Upgrade: osd.4 is safe to restart 2024-03-07T03:24:44.733 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:44 smithi148 bash[82222]: audit 2024-03-07T03:24:43.188414+0000 mgr.y (mgr.34101) 316 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-07T03:24:44.733 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:44 smithi148 bash[82222]: cephadm 2024-03-07T03:24:43.191146+0000 mgr.y (mgr.34101) 317 : cephadm 1 Upgrade: osd.4 is safe to restart 2024-03-07T03:24:45.081 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.081 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.082 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.083 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.084 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.085 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.086 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.087 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.088 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 4 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (18311 ms) 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-03-07T03:24:45.089 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:24:45.562 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:45 smithi195 bash[66542]: cluster 2024-03-07T03:24:44.432373+0000 mgr.y (mgr.34101) 318 : cluster 0 pgmap v595: 425 pgs: 425 active+clean; 11 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 301 KiB/s wr, 606 op/s; 479 KiB/s, 4 keys/s, 0 objects/s recovering 2024-03-07T03:24:45.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:45 smithi148 bash[79819]: cluster 2024-03-07T03:24:44.432373+0000 mgr.y (mgr.34101) 318 : cluster 0 pgmap v595: 425 pgs: 425 active+clean; 11 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 301 KiB/s wr, 606 op/s; 479 KiB/s, 4 keys/s, 0 objects/s recovering 2024-03-07T03:24:45.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:45 smithi148 bash[82222]: cluster 2024-03-07T03:24:44.432373+0000 mgr.y (mgr.34101) 318 : cluster 0 pgmap v595: 425 pgs: 425 active+clean; 11 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 301 KiB/s wr, 606 op/s; 479 KiB/s, 4 keys/s, 0 objects/s recovering 2024-03-07T03:24:46.556 INFO:teuthology.orchestra.run.smithi148.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:24:47.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: cephadm 2024-03-07T03:24:45.709839+0000 mgr.y (mgr.34101) 319 : cephadm 1 Upgrade: Updating osd.4 2024-03-07T03:24:47.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: audit 2024-03-07T03:24:45.719834+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: audit 2024-03-07T03:24:45.724854+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: audit 2024-03-07T03:24:45.725555+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: cephadm 2024-03-07T03:24:45.730572+0000 mgr.y (mgr.34101) 320 : cephadm 1 Deploying daemon osd.4 on smithi195 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: cluster 2024-03-07T03:24:46.433656+0000 mgr.y (mgr.34101) 321 : cluster 0 pgmap v596: 425 pgs: 425 active+clean; 11 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 270 KiB/s wr, 545 op/s; 430 KiB/s, 4 keys/s, 0 objects/s recovering 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: audit 2024-03-07T03:24:46.563944+0000 mgr.y (mgr.34101) 322 : audit 0 from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: cephadm 2024-03-07T03:24:46.564407+0000 mgr.y (mgr.34101) 323 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[79819]: audit 2024-03-07T03:24:46.569787+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: cephadm 2024-03-07T03:24:45.709839+0000 mgr.y (mgr.34101) 319 : cephadm 1 Upgrade: Updating osd.4 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: audit 2024-03-07T03:24:45.719834+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: audit 2024-03-07T03:24:45.724854+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: audit 2024-03-07T03:24:45.725555+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: cephadm 2024-03-07T03:24:45.730572+0000 mgr.y (mgr.34101) 320 : cephadm 1 Deploying daemon osd.4 on smithi195 2024-03-07T03:24:47.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: cluster 2024-03-07T03:24:46.433656+0000 mgr.y (mgr.34101) 321 : cluster 0 pgmap v596: 425 pgs: 425 active+clean; 11 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 270 KiB/s wr, 545 op/s; 430 KiB/s, 4 keys/s, 0 objects/s recovering 2024-03-07T03:24:47.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: audit 2024-03-07T03:24:46.563944+0000 mgr.y (mgr.34101) 322 : audit 0 from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:24:47.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: cephadm 2024-03-07T03:24:46.564407+0000 mgr.y (mgr.34101) 323 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:24:47.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:46 smithi148 bash[82222]: audit 2024-03-07T03:24:46.569787+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: cephadm 2024-03-07T03:24:45.709839+0000 mgr.y (mgr.34101) 319 : cephadm 1 Upgrade: Updating osd.4 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: audit 2024-03-07T03:24:45.719834+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: audit 2024-03-07T03:24:45.724854+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: audit 2024-03-07T03:24:45.725555+0000 mon.a (mon.0) 1025 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: cephadm 2024-03-07T03:24:45.730572+0000 mgr.y (mgr.34101) 320 : cephadm 1 Deploying daemon osd.4 on smithi195 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: cluster 2024-03-07T03:24:46.433656+0000 mgr.y (mgr.34101) 321 : cluster 0 pgmap v596: 425 pgs: 425 active+clean; 11 MiB data, 424 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 270 KiB/s wr, 545 op/s; 430 KiB/s, 4 keys/s, 0 objects/s recovering 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: audit 2024-03-07T03:24:46.563944+0000 mgr.y (mgr.34101) 322 : audit 0 from='client.34656 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: cephadm 2024-03-07T03:24:46.564407+0000 mgr.y (mgr.34101) 323 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:0eb4853713ff3db5d1325691ba5cce56012e1a1f886e3e0e051e4894fc066e71 2024-03-07T03:24:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:46 smithi195 bash[66542]: audit 2024-03-07T03:24:46.569787+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:47.316 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:24:47.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:24:47] "GET /metrics HTTP/1.1" 200 60734 "" "Prometheus/2.43.0" 2024-03-07T03:24:49.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:48 smithi148 bash[79819]: audit 2024-03-07T03:24:47.681811+0000 mon.a (mon.0) 1027 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:49.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:48 smithi148 bash[79819]: cluster 2024-03-07T03:24:48.435207+0000 mgr.y (mgr.34101) 324 : cluster 0 pgmap v597: 425 pgs: 425 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 226 KiB/s wr, 456 op/s; 359 KiB/s, 3 keys/s, 0 objects/s recovering 2024-03-07T03:24:49.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:48 smithi148 bash[82222]: audit 2024-03-07T03:24:47.681811+0000 mon.a (mon.0) 1027 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:49.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:48 smithi148 bash[82222]: cluster 2024-03-07T03:24:48.435207+0000 mgr.y (mgr.34101) 324 : cluster 0 pgmap v597: 425 pgs: 425 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 226 KiB/s wr, 456 op/s; 359 KiB/s, 3 keys/s, 0 objects/s recovering 2024-03-07T03:24:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:48 smithi195 bash[66542]: audit 2024-03-07T03:24:47.681811+0000 mon.a (mon.0) 1027 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:49.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:48 smithi195 bash[66542]: cluster 2024-03-07T03:24:48.435207+0000 mgr.y (mgr.34101) 324 : cluster 0 pgmap v597: 425 pgs: 425 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 226 KiB/s wr, 456 op/s; 359 KiB/s, 3 keys/s, 0 objects/s recovering 2024-03-07T03:24:50.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.313 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.313 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.314 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.314 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.314 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.315 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.315 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.315 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:50.813 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:50 smithi195 systemd[1]: Stopping Ceph osd.4 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:24:50.813 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:50 smithi195 bash[26790]: debug 2024-03-07T03:24:50.495+0000 7f0b97422700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:24:50.813 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:50 smithi195 bash[26790]: debug 2024-03-07T03:24:50.495+0000 7f0b97422700 -1 osd.4 652 *** Got signal Terminated *** 2024-03-07T03:24:50.813 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:50 smithi195 bash[26790]: debug 2024-03-07T03:24:50.495+0000 7f0b97422700 -1 osd.4 652 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:24:51.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:51 smithi148 bash[82222]: cluster 2024-03-07T03:24:50.436563+0000 mgr.y (mgr.34101) 325 : cluster 0 pgmap v598: 425 pgs: 425 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 915 KiB/s rd, 81 KiB/s wr, 187 op/s; 341 KiB/s, 0 objects/s recovering 2024-03-07T03:24:51.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:51 smithi148 bash[82222]: cluster 2024-03-07T03:24:50.500543+0000 mon.a (mon.0) 1028 : cluster 1 osd.4 marked itself down and dead 2024-03-07T03:24:51.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:51 smithi148 bash[79819]: cluster 2024-03-07T03:24:50.436563+0000 mgr.y (mgr.34101) 325 : cluster 0 pgmap v598: 425 pgs: 425 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 915 KiB/s rd, 81 KiB/s wr, 187 op/s; 341 KiB/s, 0 objects/s recovering 2024-03-07T03:24:51.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:51 smithi148 bash[79819]: cluster 2024-03-07T03:24:50.500543+0000 mon.a (mon.0) 1028 : cluster 1 osd.4 marked itself down and dead 2024-03-07T03:24:51.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:51 smithi148 bash[79819]: debug 2024-03-07T03:24:51.487+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:51.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:51 smithi195 bash[66542]: cluster 2024-03-07T03:24:50.436563+0000 mgr.y (mgr.34101) 325 : cluster 0 pgmap v598: 425 pgs: 425 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 915 KiB/s rd, 81 KiB/s wr, 187 op/s; 341 KiB/s, 0 objects/s recovering 2024-03-07T03:24:51.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:51 smithi195 bash[66542]: cluster 2024-03-07T03:24:50.500543+0000 mon.a (mon.0) 1028 : cluster 1 osd.4 marked itself down and dead 2024-03-07T03:24:52.041 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:24:52.339 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:52 smithi195 bash[59134]: ts=2024-03-07T03:24:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:24:52.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:52 smithi148 bash[79819]: cluster 2024-03-07T03:24:51.491831+0000 mon.a (mon.0) 1029 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:24:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:52 smithi148 bash[79819]: cluster 2024-03-07T03:24:51.491861+0000 mon.a (mon.0) 1030 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:52 smithi148 bash[79819]: cluster 2024-03-07T03:24:51.507216+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2024-03-07T03:24:52.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:52 smithi148 bash[82222]: cluster 2024-03-07T03:24:51.491831+0000 mon.a (mon.0) 1029 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:24:52.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:52 smithi148 bash[82222]: cluster 2024-03-07T03:24:51.491861+0000 mon.a (mon.0) 1030 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:52.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:52 smithi148 bash[82222]: cluster 2024-03-07T03:24:51.507216+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2024-03-07T03:24:52.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:52 smithi195 bash[66542]: cluster 2024-03-07T03:24:51.491831+0000 mon.a (mon.0) 1029 : cluster 3 Health check failed: 1 osds down (OSD_DOWN) 2024-03-07T03:24:52.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:52 smithi195 bash[66542]: cluster 2024-03-07T03:24:51.491861+0000 mon.a (mon.0) 1030 : cluster 4 Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:24:52.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:52 smithi195 bash[66542]: cluster 2024-03-07T03:24:51.507216+0000 mon.a (mon.0) 1031 : cluster 0 osdmap e653: 8 total, 7 up, 8 in 2024-03-07T03:24:53.090 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:52 smithi195 bash[71766]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:24:53.091 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:53 smithi195 bash[71835]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:24:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:53 smithi148 bash[79819]: cluster 2024-03-07T03:24:52.437429+0000 mgr.y (mgr.34101) 326 : cluster 0 pgmap v600: 425 pgs: 54 stale+active+clean, 371 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 660 KiB/s rd, 97 KiB/s wr, 225 op/s 2024-03-07T03:24:53.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:53 smithi148 bash[79819]: cluster 2024-03-07T03:24:52.511743+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2024-03-07T03:24:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:53 smithi148 bash[82222]: cluster 2024-03-07T03:24:52.437429+0000 mgr.y (mgr.34101) 326 : cluster 0 pgmap v600: 425 pgs: 54 stale+active+clean, 371 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 660 KiB/s rd, 97 KiB/s wr, 225 op/s 2024-03-07T03:24:53.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:53 smithi148 bash[82222]: cluster 2024-03-07T03:24:52.511743+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2024-03-07T03:24:53.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:53 smithi195 bash[66542]: cluster 2024-03-07T03:24:52.437429+0000 mgr.y (mgr.34101) 326 : cluster 0 pgmap v600: 425 pgs: 54 stale+active+clean, 371 active+clean; 11 MiB data, 423 MiB used, 715 GiB / 715 GiB avail; 660 KiB/s rd, 97 KiB/s wr, 225 op/s 2024-03-07T03:24:53.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:53 smithi195 bash[66542]: cluster 2024-03-07T03:24:52.511743+0000 mon.a (mon.0) 1032 : cluster 0 osdmap e654: 8 total, 7 up, 8 in 2024-03-07T03:24:54.755 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:54 smithi195 bash[66542]: cluster 2024-03-07T03:24:54.510777+0000 mon.a (mon.0) 1033 : cluster 3 Health check failed: Degraded data redundancy: 47/552 objects degraded (8.514%), 31 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:55.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:54 smithi148 bash[79819]: cluster 2024-03-07T03:24:54.510777+0000 mon.a (mon.0) 1033 : cluster 3 Health check failed: Degraded data redundancy: 47/552 objects degraded (8.514%), 31 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:55.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:54 smithi148 bash[82222]: cluster 2024-03-07T03:24:54.510777+0000 mon.a (mon.0) 1033 : cluster 3 Health check failed: Degraded data redundancy: 47/552 objects degraded (8.514%), 31 pgs degraded (PG_DEGRADED) 2024-03-07T03:24:55.535 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.536 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.536 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.536 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.537 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.537 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.537 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.4.service: Deactivated successfully. 2024-03-07T03:24:55.537 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: Stopped Ceph osd.4 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:24:55.537 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.538 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.538 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:24:55.577 INFO:teuthology.orchestra.run.smithi148.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:24:55.577 INFO:teuthology.orchestra.run.smithi148.stdout:alertmanager.a smithi148 *:9093,9094 running (14m) 23s ago 15m 14.5M - 0.25.0 c8568f914cd2 a6ee726aca40 2024-03-07T03:24:55.577 INFO:teuthology.orchestra.run.smithi148.stdout:grafana.a smithi195 *:3000 running (15m) 67s ago 15m 78.7M - 9.4.7 954c08fa6188 a09a8cb7fec4 2024-03-07T03:24:55.577 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi148.qizikc smithi148 running (13m) 23s ago 13m 16.2M - 18.2.1-718-gae75fc9d 0164bb8a7c67 d82cf82f7898 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi195.gtlmjn smithi195 running (13m) 67s ago 13m 20.1M - 18.2.1-718-gae75fc9d 0164bb8a7c67 f9ba5c67a194 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.x smithi195 *:8443,9283,8765 running (8m) 67s ago 23m 440M - 19.0.0-1905-gea9a5688 fdb83767c1ab 758d97044603 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.y smithi148 *:8443,9283,8765 running (9m) 23s ago 26m 510M - 19.0.0-1905-gea9a5688 fdb83767c1ab 85cf9ffcb609 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:mon.a smithi148 running (7m) 23s ago 26m 68.4M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 39441e18edee 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:mon.b smithi195 running (5m) 67s ago 24m 45.0M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 13f5d79dedd1 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:mon.c smithi148 running (6m) 23s ago 24m 51.7M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab a8f98b5ace69 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.a smithi148 *:9100 running (15m) 23s ago 15m 8664k - 1.5.0 0da6a335fe13 d9b895f99d5c 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.b smithi195 *:9100 running (15m) 67s ago 15m 8836k - 1.5.0 0da6a335fe13 be49ab11d18e 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:osd.0 smithi148 running (3m) 23s ago 22m 87.6M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab d8d07b2483c5 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:osd.1 smithi148 running (3m) 23s ago 22m 90.8M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 94c9361a6fa9 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:osd.2 smithi148 running (71s) 23s ago 21m 82.5M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 7b7ed5403f3e 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:osd.3 smithi148 running (27s) 23s ago 20m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 975fae26f7d7 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:osd.4 smithi195 running (19m) 67s ago 19m 138M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 5023ce93366a 2024-03-07T03:24:55.578 INFO:teuthology.orchestra.run.smithi148.stdout:osd.5 smithi195 running (18m) 67s ago 18m 115M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c850d7f8ae40 2024-03-07T03:24:55.579 INFO:teuthology.orchestra.run.smithi148.stdout:osd.6 smithi195 running (18m) 67s ago 18m 153M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 332f73f59fdc 2024-03-07T03:24:55.579 INFO:teuthology.orchestra.run.smithi148.stdout:osd.7 smithi195 running (17m) 67s ago 17m 124M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9a7e2329950b 2024-03-07T03:24:55.579 INFO:teuthology.orchestra.run.smithi148.stdout:prometheus.a smithi195 *:9095 running (8m) 67s ago 16m 54.9M - 2.43.0 a07b618ecd1d e5c6727b35a4 2024-03-07T03:24:55.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:55 smithi195 bash[66542]: cluster 2024-03-07T03:24:54.441214+0000 mgr.y (mgr.34101) 327 : cluster 0 pgmap v602: 425 pgs: 31 active+undersized+degraded, 62 active+undersized, 24 stale+active+clean, 308 active+clean; 19 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 1004 KiB/s rd, 1.1 MiB/s wr, 238 op/s; 47/552 objects degraded (8.514%) 2024-03-07T03:24:55.813 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:55 smithi195 systemd[1]: Started Ceph osd.4 for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:24:56.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:55 smithi148 bash[79819]: cluster 2024-03-07T03:24:54.441214+0000 mgr.y (mgr.34101) 327 : cluster 0 pgmap v602: 425 pgs: 31 active+undersized+degraded, 62 active+undersized, 24 stale+active+clean, 308 active+clean; 19 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 1004 KiB/s rd, 1.1 MiB/s wr, 238 op/s; 47/552 objects degraded (8.514%) 2024-03-07T03:24:56.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:55 smithi148 bash[82222]: cluster 2024-03-07T03:24:54.441214+0000 mgr.y (mgr.34101) 327 : cluster 0 pgmap v602: 425 pgs: 31 active+undersized+degraded, 62 active+undersized, 24 stale+active+clean, 308 active+clean; 19 MiB data, 439 MiB used, 715 GiB / 715 GiB avail; 1004 KiB/s rd, 1.1 MiB/s wr, 238 op/s; 47/552 objects degraded (8.514%) 2024-03-07T03:24:56.150 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:56.150 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:56.151 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:24:56.152 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.153 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.154 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.155 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:24:56.156 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 512~512 = 512 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 1536~512 = 0 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.157 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:24:56.158 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.159 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.160 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.161 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.162 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.163 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.164 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:24:56.165 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:24:56.475 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph orch ps' 2024-03-07T03:24:56.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[79819]: audit 2024-03-07T03:24:55.569633+0000 mgr.y (mgr.34101) 328 : audit 0 from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:24:56.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[79819]: audit 2024-03-07T03:24:55.598574+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[79819]: audit 2024-03-07T03:24:55.604550+0000 mon.a (mon.0) 1035 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[79819]: audit 2024-03-07T03:24:55.611892+0000 mon.a (mon.0) 1036 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[82222]: audit 2024-03-07T03:24:55.569633+0000 mgr.y (mgr.34101) 328 : audit 0 from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:24:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[82222]: audit 2024-03-07T03:24:55.598574+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[82222]: audit 2024-03-07T03:24:55.604550+0000 mon.a (mon.0) 1035 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:56 smithi148 bash[82222]: audit 2024-03-07T03:24:55.611892+0000 mon.a (mon.0) 1036 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:57.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:56 smithi195 bash[66542]: audit 2024-03-07T03:24:55.569633+0000 mgr.y (mgr.34101) 328 : audit 0 from='client.44582 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:24:57.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:56 smithi195 bash[66542]: audit 2024-03-07T03:24:55.598574+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:57.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:56 smithi195 bash[66542]: audit 2024-03-07T03:24:55.604550+0000 mon.a (mon.0) 1035 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:57.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:56 smithi195 bash[66542]: audit 2024-03-07T03:24:55.611892+0000 mon.a (mon.0) 1036 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:24:57.481 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[71989]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-07T03:24:57.481 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[71989]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-07T03:24:57.481 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[71989]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-07T03:24:57.481 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[71989]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-07T03:24:57.481 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[71989]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-03-07T03:24:57.481 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[71989]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-03-07T03:24:57.481 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[71989]: --> ceph-volume raw activate successful for osd ID: 4 2024-03-07T03:24:57.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:24:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:24:57] "GET /metrics HTTP/1.1" 200 60734 "" "Prometheus/2.43.0" 2024-03-07T03:24:57.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[66542]: cluster 2024-03-07T03:24:56.444080+0000 mgr.y (mgr.34101) 329 : cluster 0 pgmap v603: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 993 KiB/s rd, 2.1 MiB/s wr, 225 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:24:57.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[66542]: audit 2024-03-07T03:24:57.329105+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:57.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:57 smithi195 bash[66542]: audit 2024-03-07T03:24:57.331063+0000 mon.a (mon.0) 1038 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:58.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:57 smithi148 bash[79819]: cluster 2024-03-07T03:24:56.444080+0000 mgr.y (mgr.34101) 329 : cluster 0 pgmap v603: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 993 KiB/s rd, 2.1 MiB/s wr, 225 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:24:58.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:57 smithi148 bash[79819]: audit 2024-03-07T03:24:57.329105+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:57 smithi148 bash[79819]: audit 2024-03-07T03:24:57.331063+0000 mon.a (mon.0) 1038 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:57 smithi148 bash[82222]: cluster 2024-03-07T03:24:56.444080+0000 mgr.y (mgr.34101) 329 : cluster 0 pgmap v603: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 471 MiB used, 715 GiB / 715 GiB avail; 993 KiB/s rd, 2.1 MiB/s wr, 225 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:24:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:57 smithi148 bash[82222]: audit 2024-03-07T03:24:57.329105+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:24:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:57 smithi148 bash[82222]: audit 2024-03-07T03:24:57.331063+0000 mon.a (mon.0) 1038 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:24:59.563 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:59 smithi195 bash[72218]: debug 2024-03-07T03:24:59.178+0000 7f959b7a8700 1 -- 172.21.15.195:0/2047460932 <== mon.1 v2:172.21.15.195:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557f6fea5680 con 0x557f6fee1000 2024-03-07T03:25:00.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:24:59 smithi148 bash[79819]: cluster 2024-03-07T03:24:58.446812+0000 mgr.y (mgr.34101) 330 : cluster 0 pgmap v604: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 990 KiB/s rd, 2.1 MiB/s wr, 222 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:00.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:24:59 smithi148 bash[82222]: cluster 2024-03-07T03:24:58.446812+0000 mgr.y (mgr.34101) 330 : cluster 0 pgmap v604: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 990 KiB/s rd, 2.1 MiB/s wr, 222 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:00.062 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:24:59 smithi195 bash[72218]: debug 2024-03-07T03:24:59.730+0000 7f95a49407c0 -1 Falling back to public interface 2024-03-07T03:25:00.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:24:59 smithi195 bash[66542]: cluster 2024-03-07T03:24:58.446812+0000 mgr.y (mgr.34101) 330 : cluster 0 pgmap v604: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 990 KiB/s rd, 2.1 MiB/s wr, 222 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:01.221 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:25:02.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:01 smithi148 bash[79819]: cluster 2024-03-07T03:25:00.448698+0000 mgr.y (mgr.34101) 331 : cluster 0 pgmap v605: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.2 MiB/s wr, 839 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:02.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:01 smithi148 bash[82222]: cluster 2024-03-07T03:25:00.448698+0000 mgr.y (mgr.34101) 331 : cluster 0 pgmap v605: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.2 MiB/s wr, 839 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:02.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:01 smithi195 bash[66542]: cluster 2024-03-07T03:25:00.448698+0000 mgr.y (mgr.34101) 331 : cluster 0 pgmap v605: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 3.5 MiB/s rd, 2.2 MiB/s wr, 839 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:02.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:25:02 smithi195 bash[59134]: ts=2024-03-07T03:25:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:02.923 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:02 smithi195 bash[66542]: cluster 2024-03-07T03:25:02.243922+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 74/558 objects degraded (13.262%), 50 pgs degraded (PG_DEGRADED) 2024-03-07T03:25:02.926 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:02 smithi148 bash[79819]: cluster 2024-03-07T03:25:02.243922+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 74/558 objects degraded (13.262%), 50 pgs degraded (PG_DEGRADED) 2024-03-07T03:25:02.926 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:02 smithi148 bash[82222]: cluster 2024-03-07T03:25:02.243922+0000 mon.a (mon.0) 1039 : cluster 3 Health check update: Degraded data redundancy: 74/558 objects degraded (13.262%), 50 pgs degraded (PG_DEGRADED) 2024-03-07T03:25:03.313 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:25:02 smithi195 bash[72218]: debug 2024-03-07T03:25:02.930+0000 7f95a49407c0 -1 osd.4 652 log_to_monitors true 2024-03-07T03:25:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[82222]: cluster 2024-03-07T03:25:02.450018+0000 mgr.y (mgr.34101) 332 : cluster 0 pgmap v606: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 749 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[82222]: audit 2024-03-07T03:25:02.617140+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[82222]: audit 2024-03-07T03:25:02.633294+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[82222]: audit 2024-03-07T03:25:02.939585+0000 mon.b (mon.1) 3 : audit 1 from='osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:25:03.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[82222]: audit 2024-03-07T03:25:02.941153+0000 mon.a (mon.0) 1042 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:25:03.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[79819]: cluster 2024-03-07T03:25:02.450018+0000 mgr.y (mgr.34101) 332 : cluster 0 pgmap v606: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 749 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:03.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[79819]: audit 2024-03-07T03:25:02.617140+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[79819]: audit 2024-03-07T03:25:02.633294+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[79819]: audit 2024-03-07T03:25:02.939585+0000 mon.b (mon.1) 3 : audit 1 from='osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:25:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:03 smithi148 bash[79819]: audit 2024-03-07T03:25:02.941153+0000 mon.a (mon.0) 1042 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:25:03.795 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.795 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.795 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.795 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.795 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.796 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.797 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.798 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.799 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.800 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.801 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.802 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:25:03.803 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 4 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (18713 ms) 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCloneEncryption 2024-03-07T03:25:03.804 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:25:04.063 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:25:03 smithi195 bash[72218]: debug 2024-03-07T03:25:03.670+0000 7f9595d98700 -1 osd.4 652 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-07T03:25:04.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:03 smithi195 bash[66542]: cluster 2024-03-07T03:25:02.450018+0000 mgr.y (mgr.34101) 332 : cluster 0 pgmap v606: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 27 MiB data, 472 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.0 MiB/s wr, 749 op/s; 74/558 objects degraded (13.262%) 2024-03-07T03:25:04.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:03 smithi195 bash[66542]: audit 2024-03-07T03:25:02.617140+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:04.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:03 smithi195 bash[66542]: audit 2024-03-07T03:25:02.633294+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:04.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:03 smithi195 bash[66542]: audit 2024-03-07T03:25:02.939585+0000 mon.b (mon.1) 3 : audit 1 from='osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:25:04.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:03 smithi195 bash[66542]: audit 2024-03-07T03:25:02.941153+0000 mon.a (mon.0) 1042 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-07T03:25:04.743 INFO:teuthology.orchestra.run.smithi148.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-07T03:25:04.743 INFO:teuthology.orchestra.run.smithi148.stdout:alertmanager.a smithi148 *:9093,9094 running (15m) 32s ago 15m 14.5M - 0.25.0 c8568f914cd2 a6ee726aca40 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:grafana.a smithi195 *:3000 running (15m) 2s ago 15m 79.3M - 9.4.7 954c08fa6188 a09a8cb7fec4 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi148.qizikc smithi148 running (13m) 32s ago 13m 16.2M - 18.2.1-718-gae75fc9d 0164bb8a7c67 d82cf82f7898 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:mds.foo.smithi195.gtlmjn smithi195 running (13m) 2s ago 13m 20.5M - 18.2.1-718-gae75fc9d 0164bb8a7c67 f9ba5c67a194 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.x smithi195 *:8443,9283,8765 running (8m) 2s ago 23m 441M - 19.0.0-1905-gea9a5688 fdb83767c1ab 758d97044603 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:mgr.y smithi148 *:8443,9283,8765 running (9m) 32s ago 26m 510M - 19.0.0-1905-gea9a5688 fdb83767c1ab 85cf9ffcb609 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:mon.a smithi148 running (8m) 32s ago 26m 68.4M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 39441e18edee 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:mon.b smithi195 running (5m) 2s ago 24m 51.2M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab 13f5d79dedd1 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:mon.c smithi148 running (6m) 32s ago 24m 51.7M 2048M 19.0.0-1905-gea9a5688 fdb83767c1ab a8f98b5ace69 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.a smithi148 *:9100 running (16m) 32s ago 16m 8664k - 1.5.0 0da6a335fe13 d9b895f99d5c 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:node-exporter.b smithi195 *:9100 running (16m) 2s ago 16m 8911k - 1.5.0 0da6a335fe13 be49ab11d18e 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:osd.0 smithi148 running (4m) 32s ago 23m 87.6M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab d8d07b2483c5 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:osd.1 smithi148 running (3m) 32s ago 22m 90.8M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 94c9361a6fa9 2024-03-07T03:25:04.744 INFO:teuthology.orchestra.run.smithi148.stdout:osd.2 smithi148 running (80s) 32s ago 21m 82.5M 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 7b7ed5403f3e 2024-03-07T03:25:04.745 INFO:teuthology.orchestra.run.smithi148.stdout:osd.3 smithi148 running (36s) 32s ago 20m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 975fae26f7d7 2024-03-07T03:25:04.745 INFO:teuthology.orchestra.run.smithi148.stdout:osd.4 smithi195 running (6s) 2s ago 19m - 2524M 19.0.0-1905-gea9a5688 fdb83767c1ab 7c9925720dc8 2024-03-07T03:25:04.745 INFO:teuthology.orchestra.run.smithi148.stdout:osd.5 smithi195 running (19m) 2s ago 19m 121M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 c850d7f8ae40 2024-03-07T03:25:04.745 INFO:teuthology.orchestra.run.smithi148.stdout:osd.6 smithi195 running (18m) 2s ago 18m 151M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 332f73f59fdc 2024-03-07T03:25:04.745 INFO:teuthology.orchestra.run.smithi148.stdout:osd.7 smithi195 running (17m) 2s ago 17m 131M 2524M 18.2.1-718-gae75fc9d 0164bb8a7c67 9a7e2329950b 2024-03-07T03:25:04.745 INFO:teuthology.orchestra.run.smithi148.stdout:prometheus.a smithi195 *:9095 running (8m) 2s ago 16m 56.3M - 2.43.0 a07b618ecd1d e5c6727b35a4 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[82222]: audit 2024-03-07T03:25:03.634471+0000 mon.a (mon.0) 1043 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[82222]: audit 2024-03-07T03:25:03.637764+0000 mon.b (mon.1) 4 : audit 1 from='osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[82222]: cluster 2024-03-07T03:25:03.639578+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e655: 8 total, 7 up, 8 in 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[82222]: audit 2024-03-07T03:25:03.641030+0000 mon.a (mon.0) 1045 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[82222]: audit 2024-03-07T03:25:03.647667+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[82222]: audit 2024-03-07T03:25:03.657765+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[82222]: cluster 2024-03-07T03:25:04.452120+0000 mgr.y (mgr.34101) 333 : cluster 0 pgmap v608: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 649 op/s; 75/561 objects degraded (13.369%) 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: audit 2024-03-07T03:25:03.634471+0000 mon.a (mon.0) 1043 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: audit 2024-03-07T03:25:03.637764+0000 mon.b (mon.1) 4 : audit 1 from='osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:25:04.953 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: cluster 2024-03-07T03:25:03.639578+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e655: 8 total, 7 up, 8 in 2024-03-07T03:25:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: audit 2024-03-07T03:25:03.641030+0000 mon.a (mon.0) 1045 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:25:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: audit 2024-03-07T03:25:03.647667+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: audit 2024-03-07T03:25:03.657765+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: cluster 2024-03-07T03:25:04.452120+0000 mgr.y (mgr.34101) 333 : cluster 0 pgmap v608: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 649 op/s; 75/561 objects degraded (13.369%) 2024-03-07T03:25:04.954 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:04 smithi148 bash[79819]: debug 2024-03-07T03:25:04.650+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:25:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:04 smithi195 bash[66542]: audit 2024-03-07T03:25:03.634471+0000 mon.a (mon.0) 1043 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-07T03:25:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:04 smithi195 bash[66542]: audit 2024-03-07T03:25:03.637764+0000 mon.b (mon.1) 4 : audit 1 from='osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:25:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:04 smithi195 bash[66542]: cluster 2024-03-07T03:25:03.639578+0000 mon.a (mon.0) 1044 : cluster 0 osdmap e655: 8 total, 7 up, 8 in 2024-03-07T03:25:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:04 smithi195 bash[66542]: audit 2024-03-07T03:25:03.641030+0000 mon.a (mon.0) 1045 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi195", "root=default"]}]: dispatch 2024-03-07T03:25:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:04 smithi195 bash[66542]: audit 2024-03-07T03:25:03.647667+0000 mon.a (mon.0) 1046 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:04 smithi195 bash[66542]: audit 2024-03-07T03:25:03.657765+0000 mon.a (mon.0) 1047 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:04 smithi195 bash[66542]: cluster 2024-03-07T03:25:04.452120+0000 mgr.y (mgr.34101) 333 : cluster 0 pgmap v608: 425 pgs: 50 active+undersized+degraded, 115 active+undersized, 260 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 1.1 MiB/s wr, 649 op/s; 75/561 objects degraded (13.369%) 2024-03-07T03:25:05.594 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -e sha1=ea9a5688d9980e4ddedb59679d2960bcbac6fc71 -- bash -c 'ceph versions' 2024-03-07T03:25:06.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[82222]: cluster 2024-03-07T03:25:04.656040+0000 mon.a (mon.0) 1048 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:25:06.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[82222]: cluster 2024-03-07T03:25:04.656089+0000 mon.a (mon.0) 1049 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:25:06.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[82222]: cluster 2024-03-07T03:25:04.673790+0000 mon.a (mon.0) 1050 : cluster 1 osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799] boot 2024-03-07T03:25:06.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[82222]: cluster 2024-03-07T03:25:04.674235+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e656: 8 total, 8 up, 8 in 2024-03-07T03:25:06.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[82222]: audit 2024-03-07T03:25:04.686238+0000 mon.a (mon.0) 1052 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:25:06.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[82222]: audit 2024-03-07T03:25:04.708078+0000 mgr.y (mgr.34101) 334 : audit 0 from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:25:06.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[79819]: cluster 2024-03-07T03:25:04.656040+0000 mon.a (mon.0) 1048 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:25:06.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[79819]: cluster 2024-03-07T03:25:04.656089+0000 mon.a (mon.0) 1049 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:25:06.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[79819]: cluster 2024-03-07T03:25:04.673790+0000 mon.a (mon.0) 1050 : cluster 1 osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799] boot 2024-03-07T03:25:06.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[79819]: cluster 2024-03-07T03:25:04.674235+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e656: 8 total, 8 up, 8 in 2024-03-07T03:25:06.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[79819]: audit 2024-03-07T03:25:04.686238+0000 mon.a (mon.0) 1052 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:25:06.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:05 smithi148 bash[79819]: audit 2024-03-07T03:25:04.708078+0000 mgr.y (mgr.34101) 334 : audit 0 from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:25:06.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:05 smithi195 bash[66542]: cluster 2024-03-07T03:25:04.656040+0000 mon.a (mon.0) 1048 : cluster 4 Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-03-07T03:25:06.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:05 smithi195 bash[66542]: cluster 2024-03-07T03:25:04.656089+0000 mon.a (mon.0) 1049 : cluster 1 Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-07T03:25:06.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:05 smithi195 bash[66542]: cluster 2024-03-07T03:25:04.673790+0000 mon.a (mon.0) 1050 : cluster 1 osd.4 [v2:172.21.15.195:6800/4067214799,v1:172.21.15.195:6801/4067214799] boot 2024-03-07T03:25:06.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:05 smithi195 bash[66542]: cluster 2024-03-07T03:25:04.674235+0000 mon.a (mon.0) 1051 : cluster 0 osdmap e656: 8 total, 8 up, 8 in 2024-03-07T03:25:06.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:05 smithi195 bash[66542]: audit 2024-03-07T03:25:04.686238+0000 mon.a (mon.0) 1052 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-07T03:25:06.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:05 smithi195 bash[66542]: audit 2024-03-07T03:25:04.708078+0000 mgr.y (mgr.34101) 334 : audit 0 from='client.34671 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-07T03:25:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:06 smithi148 bash[79819]: cluster 2024-03-07T03:25:05.674897+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e657: 8 total, 8 up, 8 in 2024-03-07T03:25:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:06 smithi148 bash[79819]: cluster 2024-03-07T03:25:06.453892+0000 mgr.y (mgr.34101) 335 : cluster 0 pgmap v611: 425 pgs: 31 peering, 39 active+undersized+degraded, 78 active+undersized, 277 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 38 KiB/s wr, 155 op/s; 60/570 objects degraded (10.526%) 2024-03-07T03:25:07.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:06 smithi148 bash[82222]: cluster 2024-03-07T03:25:05.674897+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e657: 8 total, 8 up, 8 in 2024-03-07T03:25:07.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:06 smithi148 bash[82222]: cluster 2024-03-07T03:25:06.453892+0000 mgr.y (mgr.34101) 335 : cluster 0 pgmap v611: 425 pgs: 31 peering, 39 active+undersized+degraded, 78 active+undersized, 277 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 38 KiB/s wr, 155 op/s; 60/570 objects degraded (10.526%) 2024-03-07T03:25:07.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:06 smithi195 bash[66542]: cluster 2024-03-07T03:25:05.674897+0000 mon.a (mon.0) 1053 : cluster 0 osdmap e657: 8 total, 8 up, 8 in 2024-03-07T03:25:07.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:06 smithi195 bash[66542]: cluster 2024-03-07T03:25:06.453892+0000 mgr.y (mgr.34101) 335 : cluster 0 pgmap v611: 425 pgs: 31 peering, 39 active+undersized+degraded, 78 active+undersized, 277 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 38 KiB/s wr, 155 op/s; 60/570 objects degraded (10.526%) 2024-03-07T03:25:07.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:25:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:25:07] "GET /metrics HTTP/1.1" 200 60737 "" "Prometheus/2.43.0" 2024-03-07T03:25:08.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:07 smithi148 bash[79819]: cluster 2024-03-07T03:25:07.245202+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 60/570 objects degraded (10.526%), 39 pgs degraded (PG_DEGRADED) 2024-03-07T03:25:08.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:07 smithi148 bash[82222]: cluster 2024-03-07T03:25:07.245202+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 60/570 objects degraded (10.526%), 39 pgs degraded (PG_DEGRADED) 2024-03-07T03:25:08.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:07 smithi195 bash[66542]: cluster 2024-03-07T03:25:07.245202+0000 mon.a (mon.0) 1054 : cluster 3 Health check update: Degraded data redundancy: 60/570 objects degraded (10.526%), 39 pgs degraded (PG_DEGRADED) 2024-03-07T03:25:09.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:08 smithi148 bash[79819]: cluster 2024-03-07T03:25:08.456211+0000 mgr.y (mgr.34101) 336 : cluster 0 pgmap v612: 425 pgs: 31 peering, 26 active+undersized+degraded, 44 active+undersized, 324 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 39 KiB/s wr, 161 op/s; 41/576 objects degraded (7.118%); 0 B/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:25:09.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:08 smithi148 bash[82222]: cluster 2024-03-07T03:25:08.456211+0000 mgr.y (mgr.34101) 336 : cluster 0 pgmap v612: 425 pgs: 31 peering, 26 active+undersized+degraded, 44 active+undersized, 324 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 39 KiB/s wr, 161 op/s; 41/576 objects degraded (7.118%); 0 B/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:25:09.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:08 smithi195 bash[66542]: cluster 2024-03-07T03:25:08.456211+0000 mgr.y (mgr.34101) 336 : cluster 0 pgmap v612: 425 pgs: 31 peering, 26 active+undersized+degraded, 44 active+undersized, 324 active+clean; 31 MiB data, 447 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 39 KiB/s wr, 161 op/s; 41/576 objects degraded (7.118%); 0 B/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:25:09.063 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:25:08 smithi195 bash[49044]: logger=sqlstore.transactions t=2024-03-07T03:25:08.644752106Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-07T03:25:10.345 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:25:10.770 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:10 smithi195 bash[66542]: cluster 2024-03-07T03:25:10.511619+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/576 objects degraded (7.118%), 26 pgs degraded) 2024-03-07T03:25:10.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:10 smithi148 bash[79819]: cluster 2024-03-07T03:25:10.511619+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/576 objects degraded (7.118%), 26 pgs degraded) 2024-03-07T03:25:10.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:10 smithi148 bash[82222]: cluster 2024-03-07T03:25:10.511619+0000 mon.a (mon.0) 1055 : cluster 1 Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 41/576 objects degraded (7.118%), 26 pgs degraded) 2024-03-07T03:25:11.709 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:11 smithi148 bash[79819]: cluster 2024-03-07T03:25:10.458547+0000 mgr.y (mgr.34101) 337 : cluster 0 pgmap v613: 425 pgs: 425 active+clean; 31 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 35 KiB/s wr, 170 op/s; 32 KiB/s, 8 keys/s, 0 objects/s recovering 2024-03-07T03:25:11.710 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:11 smithi148 bash[82222]: cluster 2024-03-07T03:25:10.458547+0000 mgr.y (mgr.34101) 337 : cluster 0 pgmap v613: 425 pgs: 425 active+clean; 31 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 35 KiB/s wr, 170 op/s; 32 KiB/s, 8 keys/s, 0 objects/s recovering 2024-03-07T03:25:11.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:11 smithi195 bash[66542]: cluster 2024-03-07T03:25:10.458547+0000 mgr.y (mgr.34101) 337 : cluster 0 pgmap v613: 425 pgs: 425 active+clean; 31 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 35 KiB/s wr, 170 op/s; 32 KiB/s, 8 keys/s, 0 objects/s recovering 2024-03-07T03:25:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:25:12 smithi195 bash[59134]: ts=2024-03-07T03:25:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:12.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:12 smithi195 bash[66542]: cluster 2024-03-07T03:25:11.576918+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e658: 8 total, 8 up, 8 in 2024-03-07T03:25:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:12 smithi195 bash[66542]: audit 2024-03-07T03:25:12.318092+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:12 smithi195 bash[66542]: audit 2024-03-07T03:25:12.321348+0000 mon.a (mon.0) 1058 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:12 smithi148 bash[79819]: cluster 2024-03-07T03:25:11.576918+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e658: 8 total, 8 up, 8 in 2024-03-07T03:25:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:12 smithi148 bash[79819]: audit 2024-03-07T03:25:12.318092+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:12 smithi148 bash[79819]: audit 2024-03-07T03:25:12.321348+0000 mon.a (mon.0) 1058 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:12 smithi148 bash[82222]: cluster 2024-03-07T03:25:11.576918+0000 mon.a (mon.0) 1056 : cluster 0 osdmap e658: 8 total, 8 up, 8 in 2024-03-07T03:25:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:12 smithi148 bash[82222]: audit 2024-03-07T03:25:12.318092+0000 mon.a (mon.0) 1057 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:12 smithi148 bash[82222]: audit 2024-03-07T03:25:12.321348+0000 mon.a (mon.0) 1058 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout:{ 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: "mon": { 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 3 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: }, 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: "mgr": { 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 2 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: }, 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: "osd": { 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)": 3, 2024-03-07T03:25:13.602 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 5 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: }, 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: "mds": { 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)": 2 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: }, 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: "overall": { 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 18.2.1-718-gae75fc9d (ae75fc9dac756d7830c8c8f766fd136a5582707d) reef (stable)": 5, 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: "ceph version 19.0.0-1905-gea9a5688 (ea9a5688d9980e4ddedb59679d2960bcbac6fc71) squid (dev)": 10 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout: } 2024-03-07T03:25:13.603 INFO:teuthology.orchestra.run.smithi148.stdout:} 2024-03-07T03:25:14.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:13 smithi148 bash[79819]: cluster 2024-03-07T03:25:12.459175+0000 mgr.y (mgr.34101) 338 : cluster 0 pgmap v615: 425 pgs: 425 active+clean; 31 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 593 KiB/s rd, 2.3 KiB/s wr, 41 op/s; 28 KiB/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:25:14.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:13 smithi148 bash[79819]: audit 2024-03-07T03:25:13.421203+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]: dispatch 2024-03-07T03:25:14.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:13 smithi148 bash[79819]: audit 2024-03-07T03:25:13.421414+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]: dispatch 2024-03-07T03:25:14.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:13 smithi148 bash[82222]: cluster 2024-03-07T03:25:12.459175+0000 mgr.y (mgr.34101) 338 : cluster 0 pgmap v615: 425 pgs: 425 active+clean; 31 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 593 KiB/s rd, 2.3 KiB/s wr, 41 op/s; 28 KiB/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:25:14.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:13 smithi148 bash[82222]: audit 2024-03-07T03:25:13.421203+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]: dispatch 2024-03-07T03:25:14.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:13 smithi148 bash[82222]: audit 2024-03-07T03:25:13.421414+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]: dispatch 2024-03-07T03:25:14.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:13 smithi195 bash[66542]: cluster 2024-03-07T03:25:12.459175+0000 mgr.y (mgr.34101) 338 : cluster 0 pgmap v615: 425 pgs: 425 active+clean; 31 MiB data, 448 MiB used, 715 GiB / 715 GiB avail; 593 KiB/s rd, 2.3 KiB/s wr, 41 op/s; 28 KiB/s, 7 keys/s, 0 objects/s recovering 2024-03-07T03:25:14.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:13 smithi195 bash[66542]: audit 2024-03-07T03:25:13.421203+0000 mon.a (mon.0) 1059 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]: dispatch 2024-03-07T03:25:14.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:13 smithi195 bash[66542]: audit 2024-03-07T03:25:13.421414+0000 mon.a (mon.0) 1060 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]: dispatch 2024-03-07T03:25:14.304 DEBUG:teuthology.parallel:result is None 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:13.615336+0000 mon.a (mon.0) 1061 : audit 0 from='client.? 172.21.15.148:0/2045048724' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.216736+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.224660+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.226155+0000 mon.a (mon.0) 1064 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.227177+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.234534+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.314316+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.316857+0000 mon.a (mon.0) 1068 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.318471+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.319566+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.319776+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: cluster 2024-03-07T03:25:14.323053+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e659: 8 total, 8 up, 8 in 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[79819]: audit 2024-03-07T03:25:14.333378+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:13.615336+0000 mon.a (mon.0) 1061 : audit 0 from='client.? 172.21.15.148:0/2045048724' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.216736+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.224660+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.226155+0000 mon.a (mon.0) 1064 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.227177+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.234534+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.314316+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.316857+0000 mon.a (mon.0) 1068 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.318471+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:15.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.319566+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-03-07T03:25:15.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.319776+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-03-07T03:25:15.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: cluster 2024-03-07T03:25:14.323053+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e659: 8 total, 8 up, 8 in 2024-03-07T03:25:15.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:14 smithi148 bash[82222]: audit 2024-03-07T03:25:14.333378+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:13.615336+0000 mon.a (mon.0) 1061 : audit 0 from='client.? 172.21.15.148:0/2045048724' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.216736+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.224660+0000 mon.a (mon.0) 1063 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.226155+0000 mon.a (mon.0) 1064 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.227177+0000 mon.a (mon.0) 1065 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.234534+0000 mon.a (mon.0) 1066 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.314316+0000 mon.a (mon.0) 1067 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.316857+0000 mon.a (mon.0) 1068 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:25:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.318471+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:25:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.319566+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.5", "id": [7, 0]}]': finished 2024-03-07T03:25:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.319776+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-03-07T03:25:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: cluster 2024-03-07T03:25:14.323053+0000 mon.a (mon.0) 1072 : cluster 0 osdmap e659: 8 total, 8 up, 8 in 2024-03-07T03:25:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:14 smithi195 bash[66542]: audit 2024-03-07T03:25:14.333378+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:16.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:15 smithi148 bash[79819]: cluster 2024-03-07T03:25:14.459864+0000 mgr.y (mgr.34101) 339 : cluster 0 pgmap v617: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 515 KiB/s wr, 52 op/s; 27 KiB/s, 6 keys/s, 0 objects/s recovering 2024-03-07T03:25:16.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:15 smithi148 bash[79819]: cluster 2024-03-07T03:25:15.343665+0000 mon.a (mon.0) 1074 : cluster 0 osdmap e660: 8 total, 8 up, 8 in 2024-03-07T03:25:16.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:15 smithi148 bash[82222]: cluster 2024-03-07T03:25:14.459864+0000 mgr.y (mgr.34101) 339 : cluster 0 pgmap v617: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 515 KiB/s wr, 52 op/s; 27 KiB/s, 6 keys/s, 0 objects/s recovering 2024-03-07T03:25:16.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:15 smithi148 bash[82222]: cluster 2024-03-07T03:25:15.343665+0000 mon.a (mon.0) 1074 : cluster 0 osdmap e660: 8 total, 8 up, 8 in 2024-03-07T03:25:16.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:15 smithi195 bash[66542]: cluster 2024-03-07T03:25:14.459864+0000 mgr.y (mgr.34101) 339 : cluster 0 pgmap v617: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 76 KiB/s rd, 515 KiB/s wr, 52 op/s; 27 KiB/s, 6 keys/s, 0 objects/s recovering 2024-03-07T03:25:16.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:15 smithi195 bash[66542]: cluster 2024-03-07T03:25:15.343665+0000 mon.a (mon.0) 1074 : cluster 0 osdmap e660: 8 total, 8 up, 8 in 2024-03-07T03:25:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:25:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:25:17] "GET /metrics HTTP/1.1" 200 60734 "" "Prometheus/2.43.0" 2024-03-07T03:25:18.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:17 smithi148 bash[79819]: cluster 2024-03-07T03:25:16.460865+0000 mgr.y (mgr.34101) 340 : cluster 0 pgmap v619: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 685 KiB/s wr, 50 op/s 2024-03-07T03:25:18.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:17 smithi148 bash[82222]: cluster 2024-03-07T03:25:16.460865+0000 mgr.y (mgr.34101) 340 : cluster 0 pgmap v619: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 685 KiB/s wr, 50 op/s 2024-03-07T03:25:18.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:17 smithi195 bash[66542]: cluster 2024-03-07T03:25:16.460865+0000 mgr.y (mgr.34101) 340 : cluster 0 pgmap v619: 425 pgs: 425 active+clean; 35 MiB data, 460 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 685 KiB/s wr, 50 op/s 2024-03-07T03:25:19.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:18 smithi148 bash[82222]: audit 2024-03-07T03:25:17.718965+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:19.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:18 smithi148 bash[82222]: cluster 2024-03-07T03:25:18.462167+0000 mgr.y (mgr.34101) 341 : cluster 0 pgmap v620: 425 pgs: 425 active+clean; 43 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 1.7 MiB/s wr, 45 op/s 2024-03-07T03:25:19.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:18 smithi148 bash[79819]: audit 2024-03-07T03:25:17.718965+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:19.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:18 smithi148 bash[79819]: cluster 2024-03-07T03:25:18.462167+0000 mgr.y (mgr.34101) 341 : cluster 0 pgmap v620: 425 pgs: 425 active+clean; 43 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 1.7 MiB/s wr, 45 op/s 2024-03-07T03:25:19.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:18 smithi195 bash[66542]: audit 2024-03-07T03:25:17.718965+0000 mon.a (mon.0) 1075 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:25:19.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:18 smithi195 bash[66542]: cluster 2024-03-07T03:25:18.462167+0000 mgr.y (mgr.34101) 341 : cluster 0 pgmap v620: 425 pgs: 425 active+clean; 43 MiB data, 476 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 1.7 MiB/s wr, 45 op/s 2024-03-07T03:25:21.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:21 smithi148 bash[79819]: cluster 2024-03-07T03:25:20.464669+0000 mgr.y (mgr.34101) 342 : cluster 0 pgmap v621: 425 pgs: 425 active+clean; 43 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 1.5 MiB/s wr, 43 op/s 2024-03-07T03:25:21.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:21 smithi148 bash[82222]: cluster 2024-03-07T03:25:20.464669+0000 mgr.y (mgr.34101) 342 : cluster 0 pgmap v621: 425 pgs: 425 active+clean; 43 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 1.5 MiB/s wr, 43 op/s 2024-03-07T03:25:21.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:21 smithi195 bash[66542]: cluster 2024-03-07T03:25:20.464669+0000 mgr.y (mgr.34101) 342 : cluster 0 pgmap v621: 425 pgs: 425 active+clean; 43 MiB data, 500 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s rd, 1.5 MiB/s wr, 43 op/s 2024-03-07T03:25:22.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:22 smithi195 bash[66542]: cluster 2024-03-07T03:25:21.536486+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e661: 8 total, 8 up, 8 in 2024-03-07T03:25:22.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:25:22 smithi195 bash[59134]: ts=2024-03-07T03:25:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:22.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:22 smithi148 bash[79819]: cluster 2024-03-07T03:25:21.536486+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e661: 8 total, 8 up, 8 in 2024-03-07T03:25:22.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:22 smithi148 bash[82222]: cluster 2024-03-07T03:25:21.536486+0000 mon.a (mon.0) 1076 : cluster 0 osdmap e661: 8 total, 8 up, 8 in 2024-03-07T03:25:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:23 smithi148 bash[79819]: cluster 2024-03-07T03:25:22.465551+0000 mgr.y (mgr.34101) 343 : cluster 0 pgmap v623: 425 pgs: 425 active+clean; 51 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-03-07T03:25:23.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:23 smithi148 bash[82222]: cluster 2024-03-07T03:25:22.465551+0000 mgr.y (mgr.34101) 343 : cluster 0 pgmap v623: 425 pgs: 425 active+clean; 51 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-03-07T03:25:23.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:23 smithi195 bash[66542]: cluster 2024-03-07T03:25:22.465551+0000 mgr.y (mgr.34101) 343 : cluster 0 pgmap v623: 425 pgs: 425 active+clean; 51 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-03-07T03:25:25.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:25 smithi148 bash[79819]: cluster 2024-03-07T03:25:24.467377+0000 mgr.y (mgr.34101) 344 : cluster 0 pgmap v624: 425 pgs: 425 active+clean; 55 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 2.2 MiB/s wr, 39 op/s 2024-03-07T03:25:25.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:25 smithi148 bash[82222]: cluster 2024-03-07T03:25:24.467377+0000 mgr.y (mgr.34101) 344 : cluster 0 pgmap v624: 425 pgs: 425 active+clean; 55 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 2.2 MiB/s wr, 39 op/s 2024-03-07T03:25:25.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:25 smithi195 bash[66542]: cluster 2024-03-07T03:25:24.467377+0000 mgr.y (mgr.34101) 344 : cluster 0 pgmap v624: 425 pgs: 425 active+clean; 55 MiB data, 512 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 2.2 MiB/s wr, 39 op/s 2024-03-07T03:25:27.536 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:25:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:25:27] "GET /metrics HTTP/1.1" 200 60734 "" "Prometheus/2.43.0" 2024-03-07T03:25:27.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:27 smithi148 bash[79819]: cluster 2024-03-07T03:25:26.468745+0000 mgr.y (mgr.34101) 345 : cluster 0 pgmap v625: 425 pgs: 425 active+clean; 55 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-03-07T03:25:27.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:27 smithi148 bash[79819]: audit 2024-03-07T03:25:27.315840+0000 mon.a (mon.0) 1077 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:27.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:27 smithi148 bash[82222]: cluster 2024-03-07T03:25:26.468745+0000 mgr.y (mgr.34101) 345 : cluster 0 pgmap v625: 425 pgs: 425 active+clean; 55 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-03-07T03:25:27.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:27 smithi148 bash[82222]: audit 2024-03-07T03:25:27.315840+0000 mon.a (mon.0) 1077 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:27.812 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:27 smithi195 bash[66542]: cluster 2024-03-07T03:25:26.468745+0000 mgr.y (mgr.34101) 345 : cluster 0 pgmap v625: 425 pgs: 425 active+clean; 55 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-03-07T03:25:27.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:27 smithi195 bash[66542]: audit 2024-03-07T03:25:27.315840+0000 mon.a (mon.0) 1077 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:29.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:29 smithi148 bash[79819]: cluster 2024-03-07T03:25:28.470035+0000 mgr.y (mgr.34101) 346 : cluster 0 pgmap v626: 425 pgs: 425 active+clean; 55 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 1.2 MiB/s wr, 40 op/s 2024-03-07T03:25:29.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:29 smithi148 bash[82222]: cluster 2024-03-07T03:25:28.470035+0000 mgr.y (mgr.34101) 346 : cluster 0 pgmap v626: 425 pgs: 425 active+clean; 55 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 1.2 MiB/s wr, 40 op/s 2024-03-07T03:25:29.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:29 smithi195 bash[66542]: cluster 2024-03-07T03:25:28.470035+0000 mgr.y (mgr.34101) 346 : cluster 0 pgmap v626: 425 pgs: 425 active+clean; 55 MiB data, 520 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 1.2 MiB/s wr, 40 op/s 2024-03-07T03:25:31.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:31 smithi195 bash[66542]: cluster 2024-03-07T03:25:30.471594+0000 mgr.y (mgr.34101) 347 : cluster 0 pgmap v627: 425 pgs: 425 active+clean; 55 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 1.2 MiB/s wr, 36 op/s 2024-03-07T03:25:32.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:31 smithi148 bash[79819]: cluster 2024-03-07T03:25:30.471594+0000 mgr.y (mgr.34101) 347 : cluster 0 pgmap v627: 425 pgs: 425 active+clean; 55 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 1.2 MiB/s wr, 36 op/s 2024-03-07T03:25:32.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:31 smithi148 bash[82222]: cluster 2024-03-07T03:25:30.471594+0000 mgr.y (mgr.34101) 347 : cluster 0 pgmap v627: 425 pgs: 425 active+clean; 55 MiB data, 528 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 1.2 MiB/s wr, 36 op/s 2024-03-07T03:25:32.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:25:32 smithi195 bash[59134]: ts=2024-03-07T03:25:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:33.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:33 smithi195 bash[66542]: cluster 2024-03-07T03:25:32.472595+0000 mgr.y (mgr.34101) 348 : cluster 0 pgmap v628: 425 pgs: 425 active+clean; 55 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 376 KiB/s wr, 33 op/s 2024-03-07T03:25:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:33 smithi148 bash[79819]: cluster 2024-03-07T03:25:32.472595+0000 mgr.y (mgr.34101) 348 : cluster 0 pgmap v628: 425 pgs: 425 active+clean; 55 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 376 KiB/s wr, 33 op/s 2024-03-07T03:25:34.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:33 smithi148 bash[82222]: cluster 2024-03-07T03:25:32.472595+0000 mgr.y (mgr.34101) 348 : cluster 0 pgmap v628: 425 pgs: 425 active+clean; 55 MiB data, 532 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 376 KiB/s wr, 33 op/s 2024-03-07T03:25:35.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:35 smithi195 bash[66542]: cluster 2024-03-07T03:25:34.474375+0000 mgr.y (mgr.34101) 349 : cluster 0 pgmap v629: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 415 KiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:25:36.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:35 smithi148 bash[79819]: cluster 2024-03-07T03:25:34.474375+0000 mgr.y (mgr.34101) 349 : cluster 0 pgmap v629: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 415 KiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:25:36.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:35 smithi148 bash[82222]: cluster 2024-03-07T03:25:34.474375+0000 mgr.y (mgr.34101) 349 : cluster 0 pgmap v629: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 415 KiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:25:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:25:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:25:37] "GET /metrics HTTP/1.1" 200 60729 "" "Prometheus/2.43.0" 2024-03-07T03:25:37.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:37 smithi195 bash[66542]: cluster 2024-03-07T03:25:36.475731+0000 mgr.y (mgr.34101) 350 : cluster 0 pgmap v630: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-03-07T03:25:38.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:37 smithi148 bash[79819]: cluster 2024-03-07T03:25:36.475731+0000 mgr.y (mgr.34101) 350 : cluster 0 pgmap v630: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-03-07T03:25:38.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:37 smithi148 bash[82222]: cluster 2024-03-07T03:25:36.475731+0000 mgr.y (mgr.34101) 350 : cluster 0 pgmap v630: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.3 MiB/s wr, 9 op/s 2024-03-07T03:25:39.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:39 smithi195 bash[66542]: cluster 2024-03-07T03:25:38.477721+0000 mgr.y (mgr.34101) 351 : cluster 0 pgmap v631: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:25:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:39 smithi148 bash[79819]: cluster 2024-03-07T03:25:38.477721+0000 mgr.y (mgr.34101) 351 : cluster 0 pgmap v631: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:25:40.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:39 smithi148 bash[82222]: cluster 2024-03-07T03:25:38.477721+0000 mgr.y (mgr.34101) 351 : cluster 0 pgmap v631: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:25:41.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:41 smithi195 bash[66542]: cluster 2024-03-07T03:25:40.480010+0000 mgr.y (mgr.34101) 352 : cluster 0 pgmap v632: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:25:41.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:41 smithi195 bash[66542]: cluster 2024-03-07T03:25:40.575962+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e662: 8 total, 8 up, 8 in 2024-03-07T03:25:42.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:41 smithi148 bash[82222]: cluster 2024-03-07T03:25:40.480010+0000 mgr.y (mgr.34101) 352 : cluster 0 pgmap v632: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:25:42.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:41 smithi148 bash[82222]: cluster 2024-03-07T03:25:40.575962+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e662: 8 total, 8 up, 8 in 2024-03-07T03:25:42.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:41 smithi148 bash[79819]: cluster 2024-03-07T03:25:40.480010+0000 mgr.y (mgr.34101) 352 : cluster 0 pgmap v632: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 4 op/s 2024-03-07T03:25:42.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:41 smithi148 bash[79819]: cluster 2024-03-07T03:25:40.575962+0000 mon.a (mon.0) 1078 : cluster 0 osdmap e662: 8 total, 8 up, 8 in 2024-03-07T03:25:42.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:25:42 smithi195 bash[59134]: ts=2024-03-07T03:25:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:42 smithi148 bash[79819]: audit 2024-03-07T03:25:42.316129+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:42 smithi148 bash[82222]: audit 2024-03-07T03:25:42.316129+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:42 smithi195 bash[66542]: audit 2024-03-07T03:25:42.316129+0000 mon.a (mon.0) 1079 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:44.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:43 smithi148 bash[79819]: cluster 2024-03-07T03:25:42.481005+0000 mgr.y (mgr.34101) 353 : cluster 0 pgmap v634: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:25:44.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:43 smithi148 bash[82222]: cluster 2024-03-07T03:25:42.481005+0000 mgr.y (mgr.34101) 353 : cluster 0 pgmap v634: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:25:44.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:43 smithi195 bash[66542]: cluster 2024-03-07T03:25:42.481005+0000 mgr.y (mgr.34101) 353 : cluster 0 pgmap v634: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:25:46.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:45 smithi148 bash[79819]: cluster 2024-03-07T03:25:44.482682+0000 mgr.y (mgr.34101) 354 : cluster 0 pgmap v635: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 992 KiB/s rd, 1023 B/s wr, 36 op/s 2024-03-07T03:25:46.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:45 smithi148 bash[82222]: cluster 2024-03-07T03:25:44.482682+0000 mgr.y (mgr.34101) 354 : cluster 0 pgmap v635: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 992 KiB/s rd, 1023 B/s wr, 36 op/s 2024-03-07T03:25:46.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:45 smithi195 bash[66542]: cluster 2024-03-07T03:25:44.482682+0000 mgr.y (mgr.34101) 354 : cluster 0 pgmap v635: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 992 KiB/s rd, 1023 B/s wr, 36 op/s 2024-03-07T03:25:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:25:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:25:47] "GET /metrics HTTP/1.1" 200 60730 "" "Prometheus/2.43.0" 2024-03-07T03:25:48.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:47 smithi148 bash[79819]: cluster 2024-03-07T03:25:46.483745+0000 mgr.y (mgr.34101) 355 : cluster 0 pgmap v636: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-03-07T03:25:48.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:47 smithi148 bash[82222]: cluster 2024-03-07T03:25:46.483745+0000 mgr.y (mgr.34101) 355 : cluster 0 pgmap v636: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-03-07T03:25:48.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:47 smithi195 bash[66542]: cluster 2024-03-07T03:25:46.483745+0000 mgr.y (mgr.34101) 355 : cluster 0 pgmap v636: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-03-07T03:25:50.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:49 smithi148 bash[79819]: cluster 2024-03-07T03:25:48.484965+0000 mgr.y (mgr.34101) 356 : cluster 0 pgmap v637: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 183 KiB/s rd, 2.9 KiB/s wr, 50 op/s 2024-03-07T03:25:50.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:49 smithi148 bash[82222]: cluster 2024-03-07T03:25:48.484965+0000 mgr.y (mgr.34101) 356 : cluster 0 pgmap v637: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 183 KiB/s rd, 2.9 KiB/s wr, 50 op/s 2024-03-07T03:25:50.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:49 smithi195 bash[66542]: cluster 2024-03-07T03:25:48.484965+0000 mgr.y (mgr.34101) 356 : cluster 0 pgmap v637: 425 pgs: 425 active+clean; 71 MiB data, 568 MiB used, 715 GiB / 715 GiB avail; 183 KiB/s rd, 2.9 KiB/s wr, 50 op/s 2024-03-07T03:25:52.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:51 smithi148 bash[79819]: cluster 2024-03-07T03:25:50.486519+0000 mgr.y (mgr.34101) 357 : cluster 0 pgmap v638: 425 pgs: 425 active+clean; 155 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 8.4 MiB/s wr, 56 op/s 2024-03-07T03:25:52.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:51 smithi148 bash[82222]: cluster 2024-03-07T03:25:50.486519+0000 mgr.y (mgr.34101) 357 : cluster 0 pgmap v638: 425 pgs: 425 active+clean; 155 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 8.4 MiB/s wr, 56 op/s 2024-03-07T03:25:52.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:51 smithi195 bash[66542]: cluster 2024-03-07T03:25:50.486519+0000 mgr.y (mgr.34101) 357 : cluster 0 pgmap v638: 425 pgs: 425 active+clean; 155 MiB data, 856 MiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 8.4 MiB/s wr, 56 op/s 2024-03-07T03:25:52.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:25:52 smithi195 bash[59134]: ts=2024-03-07T03:25:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:25:53.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:52 smithi148 bash[79819]: cluster 2024-03-07T03:25:51.633450+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2024-03-07T03:25:53.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:52 smithi148 bash[82222]: cluster 2024-03-07T03:25:51.633450+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2024-03-07T03:25:53.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:52 smithi195 bash[66542]: cluster 2024-03-07T03:25:51.633450+0000 mon.a (mon.0) 1080 : cluster 0 osdmap e663: 8 total, 8 up, 8 in 2024-03-07T03:25:54.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:53 smithi148 bash[79819]: cluster 2024-03-07T03:25:52.487485+0000 mgr.y (mgr.34101) 358 : cluster 0 pgmap v640: 425 pgs: 425 active+clean; 155 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 189 KiB/s rd, 11 MiB/s wr, 65 op/s 2024-03-07T03:25:54.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:53 smithi148 bash[79819]: cluster 2024-03-07T03:25:52.649717+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2024-03-07T03:25:54.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:53 smithi148 bash[82222]: cluster 2024-03-07T03:25:52.487485+0000 mgr.y (mgr.34101) 358 : cluster 0 pgmap v640: 425 pgs: 425 active+clean; 155 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 189 KiB/s rd, 11 MiB/s wr, 65 op/s 2024-03-07T03:25:54.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:53 smithi148 bash[82222]: cluster 2024-03-07T03:25:52.649717+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2024-03-07T03:25:54.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:53 smithi195 bash[66542]: cluster 2024-03-07T03:25:52.487485+0000 mgr.y (mgr.34101) 358 : cluster 0 pgmap v640: 425 pgs: 425 active+clean; 155 MiB data, 928 MiB used, 714 GiB / 715 GiB avail; 189 KiB/s rd, 11 MiB/s wr, 65 op/s 2024-03-07T03:25:54.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:53 smithi195 bash[66542]: cluster 2024-03-07T03:25:52.649717+0000 mon.a (mon.0) 1081 : cluster 0 osdmap e664: 8 total, 8 up, 8 in 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:read: 4 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCloneEncryption (51082 ms) 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.LUKS1UnderLUKS2WithoutResize 2024-03-07T03:25:54.878 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:25:55.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:54 smithi148 bash[82222]: cluster 2024-03-07T03:25:53.647790+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2024-03-07T03:25:55.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:54 smithi148 bash[82222]: cluster 2024-03-07T03:25:54.489384+0000 mgr.y (mgr.34101) 359 : cluster 0 pgmap v643: 425 pgs: 425 active+clean; 91 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 43 MiB/s wr, 230 op/s 2024-03-07T03:25:55.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:54 smithi148 bash[79819]: cluster 2024-03-07T03:25:53.647790+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2024-03-07T03:25:55.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:54 smithi148 bash[79819]: cluster 2024-03-07T03:25:54.489384+0000 mgr.y (mgr.34101) 359 : cluster 0 pgmap v643: 425 pgs: 425 active+clean; 91 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 43 MiB/s wr, 230 op/s 2024-03-07T03:25:55.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:54 smithi195 bash[66542]: cluster 2024-03-07T03:25:53.647790+0000 mon.a (mon.0) 1082 : cluster 0 osdmap e665: 8 total, 8 up, 8 in 2024-03-07T03:25:55.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:54 smithi195 bash[66542]: cluster 2024-03-07T03:25:54.489384+0000 mgr.y (mgr.34101) 359 : cluster 0 pgmap v643: 425 pgs: 425 active+clean; 91 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 43 MiB/s wr, 230 op/s 2024-03-07T03:25:56.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:55 smithi148 bash[79819]: cluster 2024-03-07T03:25:54.677858+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2024-03-07T03:25:56.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:55 smithi148 bash[82222]: cluster 2024-03-07T03:25:54.677858+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2024-03-07T03:25:56.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:55 smithi195 bash[66542]: cluster 2024-03-07T03:25:54.677858+0000 mon.a (mon.0) 1083 : cluster 0 osdmap e666: 8 total, 8 up, 8 in 2024-03-07T03:25:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:56 smithi148 bash[79819]: cluster 2024-03-07T03:25:56.491381+0000 mgr.y (mgr.34101) 360 : cluster 0 pgmap v645: 425 pgs: 425 active+clean; 35 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 30 MiB/s wr, 277 op/s 2024-03-07T03:25:57.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:56 smithi148 bash[82222]: cluster 2024-03-07T03:25:56.491381+0000 mgr.y (mgr.34101) 360 : cluster 0 pgmap v645: 425 pgs: 425 active+clean; 35 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 30 MiB/s wr, 277 op/s 2024-03-07T03:25:57.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:56 smithi195 bash[66542]: cluster 2024-03-07T03:25:56.491381+0000 mgr.y (mgr.34101) 360 : cluster 0 pgmap v645: 425 pgs: 425 active+clean; 35 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s rd, 30 MiB/s wr, 277 op/s 2024-03-07T03:25:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:25:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:25:57] "GET /metrics HTTP/1.1" 200 60730 "" "Prometheus/2.43.0" 2024-03-07T03:25:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:58 smithi148 bash[82222]: cluster 2024-03-07T03:25:57.264443+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2024-03-07T03:25:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:58 smithi148 bash[82222]: audit 2024-03-07T03:25:57.318032+0000 mon.a (mon.0) 1085 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:58 smithi148 bash[79819]: cluster 2024-03-07T03:25:57.264443+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2024-03-07T03:25:58.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:58 smithi148 bash[79819]: audit 2024-03-07T03:25:57.318032+0000 mon.a (mon.0) 1085 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:58.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:58 smithi195 bash[66542]: cluster 2024-03-07T03:25:57.264443+0000 mon.a (mon.0) 1084 : cluster 0 osdmap e667: 8 total, 8 up, 8 in 2024-03-07T03:25:58.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:58 smithi195 bash[66542]: audit 2024-03-07T03:25:57.318032+0000 mon.a (mon.0) 1085 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:25:59.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:25:59 smithi148 bash[79819]: cluster 2024-03-07T03:25:58.493333+0000 mgr.y (mgr.34101) 361 : cluster 0 pgmap v647: 425 pgs: 425 active+clean; 31 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 25 MiB/s wr, 248 op/s 2024-03-07T03:25:59.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:25:59 smithi148 bash[82222]: cluster 2024-03-07T03:25:58.493333+0000 mgr.y (mgr.34101) 361 : cluster 0 pgmap v647: 425 pgs: 425 active+clean; 31 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 25 MiB/s wr, 248 op/s 2024-03-07T03:25:59.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:25:59 smithi195 bash[66542]: cluster 2024-03-07T03:25:58.493333+0000 mgr.y (mgr.34101) 361 : cluster 0 pgmap v647: 425 pgs: 425 active+clean; 31 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 25 MiB/s wr, 248 op/s 2024-03-07T03:26:00.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:00 smithi148 bash[79819]: cluster 2024-03-07T03:25:59.277122+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2024-03-07T03:26:00.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:00 smithi148 bash[82222]: cluster 2024-03-07T03:25:59.277122+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2024-03-07T03:26:00.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:00 smithi195 bash[66542]: cluster 2024-03-07T03:25:59.277122+0000 mon.a (mon.0) 1086 : cluster 0 osdmap e668: 8 total, 8 up, 8 in 2024-03-07T03:26:01.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:01 smithi148 bash[79819]: cluster 2024-03-07T03:26:00.494905+0000 mgr.y (mgr.34101) 362 : cluster 0 pgmap v649: 425 pgs: 425 active+clean; 43 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 2.0 MiB/s wr, 90 op/s 2024-03-07T03:26:01.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:01 smithi148 bash[82222]: cluster 2024-03-07T03:26:00.494905+0000 mgr.y (mgr.34101) 362 : cluster 0 pgmap v649: 425 pgs: 425 active+clean; 43 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 2.0 MiB/s wr, 90 op/s 2024-03-07T03:26:01.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:01 smithi195 bash[66542]: cluster 2024-03-07T03:26:00.494905+0000 mgr.y (mgr.34101) 362 : cluster 0 pgmap v649: 425 pgs: 425 active+clean; 43 MiB data, 466 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 2.0 MiB/s wr, 90 op/s 2024-03-07T03:26:02.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:26:02 smithi195 bash[59134]: ts=2024-03-07T03:26:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:03 smithi148 bash[79819]: cluster 2024-03-07T03:26:02.264559+0000 mon.a (mon.0) 1087 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2024-03-07T03:26:03.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:03 smithi148 bash[79819]: cluster 2024-03-07T03:26:02.495616+0000 mgr.y (mgr.34101) 363 : cluster 0 pgmap v651: 425 pgs: 425 active+clean; 47 MiB data, 478 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 2.7 MiB/s wr, 61 op/s 2024-03-07T03:26:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:03 smithi148 bash[82222]: cluster 2024-03-07T03:26:02.264559+0000 mon.a (mon.0) 1087 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2024-03-07T03:26:03.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:03 smithi148 bash[82222]: cluster 2024-03-07T03:26:02.495616+0000 mgr.y (mgr.34101) 363 : cluster 0 pgmap v651: 425 pgs: 425 active+clean; 47 MiB data, 478 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 2.7 MiB/s wr, 61 op/s 2024-03-07T03:26:03.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:03 smithi195 bash[66542]: cluster 2024-03-07T03:26:02.264559+0000 mon.a (mon.0) 1087 : cluster 0 osdmap e669: 8 total, 8 up, 8 in 2024-03-07T03:26:03.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:03 smithi195 bash[66542]: cluster 2024-03-07T03:26:02.495616+0000 mgr.y (mgr.34101) 363 : cluster 0 pgmap v651: 425 pgs: 425 active+clean; 47 MiB data, 478 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 2.7 MiB/s wr, 61 op/s 2024-03-07T03:26:05.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:05 smithi195 bash[66542]: cluster 2024-03-07T03:26:04.497472+0000 mgr.y (mgr.34101) 364 : cluster 0 pgmap v652: 425 pgs: 425 active+clean; 60 MiB data, 548 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 4.0 MiB/s wr, 83 op/s 2024-03-07T03:26:06.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:05 smithi148 bash[79819]: cluster 2024-03-07T03:26:04.497472+0000 mgr.y (mgr.34101) 364 : cluster 0 pgmap v652: 425 pgs: 425 active+clean; 60 MiB data, 548 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 4.0 MiB/s wr, 83 op/s 2024-03-07T03:26:06.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:05 smithi148 bash[82222]: cluster 2024-03-07T03:26:04.497472+0000 mgr.y (mgr.34101) 364 : cluster 0 pgmap v652: 425 pgs: 425 active+clean; 60 MiB data, 548 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 4.0 MiB/s wr, 83 op/s 2024-03-07T03:26:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:26:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:26:07] "GET /metrics HTTP/1.1" 200 60729 "" "Prometheus/2.43.0" 2024-03-07T03:26:07.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:07 smithi195 bash[66542]: cluster 2024-03-07T03:26:06.498814+0000 mgr.y (mgr.34101) 365 : cluster 0 pgmap v653: 425 pgs: 425 active+clean; 60 MiB data, 548 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.6 MiB/s wr, 62 op/s 2024-03-07T03:26:08.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:07 smithi148 bash[79819]: cluster 2024-03-07T03:26:06.498814+0000 mgr.y (mgr.34101) 365 : cluster 0 pgmap v653: 425 pgs: 425 active+clean; 60 MiB data, 548 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.6 MiB/s wr, 62 op/s 2024-03-07T03:26:08.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:07 smithi148 bash[82222]: cluster 2024-03-07T03:26:06.498814+0000 mgr.y (mgr.34101) 365 : cluster 0 pgmap v653: 425 pgs: 425 active+clean; 60 MiB data, 548 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 3.6 MiB/s wr, 62 op/s 2024-03-07T03:26:09.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:09 smithi195 bash[66542]: cluster 2024-03-07T03:26:08.500194+0000 mgr.y (mgr.34101) 366 : cluster 0 pgmap v654: 425 pgs: 425 active+clean; 64 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-03-07T03:26:10.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:09 smithi148 bash[79819]: cluster 2024-03-07T03:26:08.500194+0000 mgr.y (mgr.34101) 366 : cluster 0 pgmap v654: 425 pgs: 425 active+clean; 64 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-03-07T03:26:10.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:09 smithi148 bash[82222]: cluster 2024-03-07T03:26:08.500194+0000 mgr.y (mgr.34101) 366 : cluster 0 pgmap v654: 425 pgs: 425 active+clean; 64 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-03-07T03:26:10.783 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.LUKS1UnderLUKS2WithoutResize (15905 ms) 2024-03-07T03:26:10.783 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.LUKS2UnderLUKS1WithoutResize 2024-03-07T03:26:10.783 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:26:11.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:11 smithi195 bash[66542]: cluster 2024-03-07T03:26:10.501432+0000 mgr.y (mgr.34101) 367 : cluster 0 pgmap v655: 425 pgs: 425 active+clean; 68 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-03-07T03:26:12.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:11 smithi148 bash[79819]: cluster 2024-03-07T03:26:10.501432+0000 mgr.y (mgr.34101) 367 : cluster 0 pgmap v655: 425 pgs: 425 active+clean; 68 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-03-07T03:26:12.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:11 smithi148 bash[82222]: cluster 2024-03-07T03:26:10.501432+0000 mgr.y (mgr.34101) 367 : cluster 0 pgmap v655: 425 pgs: 425 active+clean; 68 MiB data, 588 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.5 MiB/s wr, 28 op/s 2024-03-07T03:26:12.562 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:26:12 smithi195 bash[59134]: ts=2024-03-07T03:26:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:12.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:12 smithi195 bash[66542]: audit 2024-03-07T03:26:12.317240+0000 mon.a (mon.0) 1088 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:12 smithi148 bash[79819]: audit 2024-03-07T03:26:12.317240+0000 mon.a (mon.0) 1088 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:12 smithi148 bash[82222]: audit 2024-03-07T03:26:12.317240+0000 mon.a (mon.0) 1088 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:14.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:13 smithi148 bash[79819]: cluster 2024-03-07T03:26:12.502349+0000 mgr.y (mgr.34101) 368 : cluster 0 pgmap v656: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-03-07T03:26:14.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:13 smithi148 bash[82222]: cluster 2024-03-07T03:26:12.502349+0000 mgr.y (mgr.34101) 368 : cluster 0 pgmap v656: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-03-07T03:26:14.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:13 smithi195 bash[66542]: cluster 2024-03-07T03:26:12.502349+0000 mgr.y (mgr.34101) 368 : cluster 0 pgmap v656: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 2.8 MiB/s wr, 27 op/s 2024-03-07T03:26:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:14 smithi148 bash[79819]: audit 2024-03-07T03:26:14.480469+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:26:15.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:14 smithi148 bash[82222]: audit 2024-03-07T03:26:14.480469+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:26:15.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:14 smithi195 bash[66542]: audit 2024-03-07T03:26:14.480469+0000 mon.a (mon.0) 1089 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:26:16.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:15 smithi148 bash[79819]: cluster 2024-03-07T03:26:14.504040+0000 mgr.y (mgr.34101) 369 : cluster 0 pgmap v657: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-03-07T03:26:16.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:15 smithi148 bash[82222]: cluster 2024-03-07T03:26:14.504040+0000 mgr.y (mgr.34101) 369 : cluster 0 pgmap v657: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-03-07T03:26:16.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:15 smithi195 bash[66542]: cluster 2024-03-07T03:26:14.504040+0000 mgr.y (mgr.34101) 369 : cluster 0 pgmap v657: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-03-07T03:26:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:26:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:26:17] "GET /metrics HTTP/1.1" 200 60735 "" "Prometheus/2.43.0" 2024-03-07T03:26:18.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:17 smithi148 bash[79819]: cluster 2024-03-07T03:26:16.505322+0000 mgr.y (mgr.34101) 370 : cluster 0 pgmap v658: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-03-07T03:26:18.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:17 smithi148 bash[82222]: cluster 2024-03-07T03:26:16.505322+0000 mgr.y (mgr.34101) 370 : cluster 0 pgmap v658: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-03-07T03:26:18.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:17 smithi195 bash[66542]: cluster 2024-03-07T03:26:16.505322+0000 mgr.y (mgr.34101) 370 : cluster 0 pgmap v658: 425 pgs: 425 active+clean; 76 MiB data, 596 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.3 MiB/s wr, 7 op/s 2024-03-07T03:26:20.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:19 smithi148 bash[79819]: cluster 2024-03-07T03:26:18.507423+0000 mgr.y (mgr.34101) 371 : cluster 0 pgmap v659: 425 pgs: 425 active+clean; 108 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.0 MiB/s wr, 8 op/s 2024-03-07T03:26:20.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:19 smithi148 bash[79819]: cluster 2024-03-07T03:26:18.603038+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2024-03-07T03:26:20.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:19 smithi148 bash[82222]: cluster 2024-03-07T03:26:18.507423+0000 mgr.y (mgr.34101) 371 : cluster 0 pgmap v659: 425 pgs: 425 active+clean; 108 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.0 MiB/s wr, 8 op/s 2024-03-07T03:26:20.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:19 smithi148 bash[82222]: cluster 2024-03-07T03:26:18.603038+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2024-03-07T03:26:20.062 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:19 smithi195 bash[66542]: cluster 2024-03-07T03:26:18.507423+0000 mgr.y (mgr.34101) 371 : cluster 0 pgmap v659: 425 pgs: 425 active+clean; 108 MiB data, 616 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 4.0 MiB/s wr, 8 op/s 2024-03-07T03:26:20.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:19 smithi195 bash[66542]: cluster 2024-03-07T03:26:18.603038+0000 mon.a (mon.0) 1090 : cluster 0 osdmap e670: 8 total, 8 up, 8 in 2024-03-07T03:26:21.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: audit 2024-03-07T03:26:20.276723+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: audit 2024-03-07T03:26:20.282973+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: cluster 2024-03-07T03:26:20.508849+0000 mgr.y (mgr.34101) 372 : cluster 0 pgmap v661: 425 pgs: 425 active+clean; 117 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.9 MiB/s wr, 17 op/s 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: audit 2024-03-07T03:26:20.592494+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: audit 2024-03-07T03:26:20.600762+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: audit 2024-03-07T03:26:21.231378+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: audit 2024-03-07T03:26:21.233033+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[79819]: audit 2024-03-07T03:26:21.244354+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: audit 2024-03-07T03:26:20.276723+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: audit 2024-03-07T03:26:20.282973+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: cluster 2024-03-07T03:26:20.508849+0000 mgr.y (mgr.34101) 372 : cluster 0 pgmap v661: 425 pgs: 425 active+clean; 117 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.9 MiB/s wr, 17 op/s 2024-03-07T03:26:21.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: audit 2024-03-07T03:26:20.592494+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: audit 2024-03-07T03:26:20.600762+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: audit 2024-03-07T03:26:21.231378+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:26:21.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: audit 2024-03-07T03:26:21.233033+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:26:21.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:21 smithi148 bash[82222]: audit 2024-03-07T03:26:21.244354+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: audit 2024-03-07T03:26:20.276723+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: audit 2024-03-07T03:26:20.282973+0000 mon.a (mon.0) 1092 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: cluster 2024-03-07T03:26:20.508849+0000 mgr.y (mgr.34101) 372 : cluster 0 pgmap v661: 425 pgs: 425 active+clean; 117 MiB data, 671 MiB used, 715 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.9 MiB/s wr, 17 op/s 2024-03-07T03:26:21.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: audit 2024-03-07T03:26:20.592494+0000 mon.a (mon.0) 1093 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: audit 2024-03-07T03:26:20.600762+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:21.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: audit 2024-03-07T03:26:21.231378+0000 mon.a (mon.0) 1095 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:26:21.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: audit 2024-03-07T03:26:21.233033+0000 mon.a (mon.0) 1096 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:26:21.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:21 smithi195 bash[66542]: audit 2024-03-07T03:26:21.244354+0000 mon.a (mon.0) 1097 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:26:22.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:26:22 smithi195 bash[59134]: ts=2024-03-07T03:26:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:23.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:23 smithi195 bash[66542]: cluster 2024-03-07T03:26:22.510128+0000 mgr.y (mgr.34101) 373 : cluster 0 pgmap v662: 425 pgs: 425 active+clean; 117 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 17 op/s 2024-03-07T03:26:24.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:23 smithi148 bash[79819]: cluster 2024-03-07T03:26:22.510128+0000 mgr.y (mgr.34101) 373 : cluster 0 pgmap v662: 425 pgs: 425 active+clean; 117 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 17 op/s 2024-03-07T03:26:24.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:23 smithi148 bash[82222]: cluster 2024-03-07T03:26:22.510128+0000 mgr.y (mgr.34101) 373 : cluster 0 pgmap v662: 425 pgs: 425 active+clean; 117 MiB data, 675 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 17 op/s 2024-03-07T03:26:25.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:25 smithi195 bash[66542]: cluster 2024-03-07T03:26:24.511662+0000 mgr.y (mgr.34101) 374 : cluster 0 pgmap v663: 425 pgs: 425 active+clean; 121 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-03-07T03:26:26.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:25 smithi148 bash[79819]: cluster 2024-03-07T03:26:24.511662+0000 mgr.y (mgr.34101) 374 : cluster 0 pgmap v663: 425 pgs: 425 active+clean; 121 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-03-07T03:26:26.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:25 smithi148 bash[82222]: cluster 2024-03-07T03:26:24.511662+0000 mgr.y (mgr.34101) 374 : cluster 0 pgmap v663: 425 pgs: 425 active+clean; 121 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-03-07T03:26:26.469 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.LUKS2UnderLUKS1WithoutResize (15687 ms) 2024-03-07T03:26:26.469 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EncryptionFormatNoData 2024-03-07T03:26:26.469 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:26:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:26:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:26:27] "GET /metrics HTTP/1.1" 200 60735 "" "Prometheus/2.43.0" 2024-03-07T03:26:28.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:27 smithi148 bash[79819]: cluster 2024-03-07T03:26:26.513594+0000 mgr.y (mgr.34101) 375 : cluster 0 pgmap v664: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-03-07T03:26:28.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:27 smithi148 bash[79819]: audit 2024-03-07T03:26:27.317780+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:27 smithi148 bash[82222]: cluster 2024-03-07T03:26:26.513594+0000 mgr.y (mgr.34101) 375 : cluster 0 pgmap v664: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-03-07T03:26:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:27 smithi148 bash[82222]: audit 2024-03-07T03:26:27.317780+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:28.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:27 smithi195 bash[66542]: cluster 2024-03-07T03:26:26.513594+0000 mgr.y (mgr.34101) 375 : cluster 0 pgmap v664: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-03-07T03:26:28.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:27 smithi195 bash[66542]: audit 2024-03-07T03:26:27.317780+0000 mon.a (mon.0) 1098 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:30.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:29 smithi148 bash[82222]: cluster 2024-03-07T03:26:28.515495+0000 mgr.y (mgr.34101) 376 : cluster 0 pgmap v665: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 38 op/s 2024-03-07T03:26:30.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:29 smithi148 bash[79819]: cluster 2024-03-07T03:26:28.515495+0000 mgr.y (mgr.34101) 376 : cluster 0 pgmap v665: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 38 op/s 2024-03-07T03:26:30.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:29 smithi195 bash[66542]: cluster 2024-03-07T03:26:28.515495+0000 mgr.y (mgr.34101) 376 : cluster 0 pgmap v665: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 38 op/s 2024-03-07T03:26:30.892 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.EncryptionFormatNoData (4423 ms) 2024-03-07T03:26:30.893 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EncryptionLoadBadSize 2024-03-07T03:26:30.893 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:26:32.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:31 smithi148 bash[82222]: cluster 2024-03-07T03:26:30.517809+0000 mgr.y (mgr.34101) 377 : cluster 0 pgmap v666: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 37 op/s 2024-03-07T03:26:32.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:31 smithi148 bash[79819]: cluster 2024-03-07T03:26:30.517809+0000 mgr.y (mgr.34101) 377 : cluster 0 pgmap v666: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 37 op/s 2024-03-07T03:26:32.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:31 smithi195 bash[66542]: cluster 2024-03-07T03:26:30.517809+0000 mgr.y (mgr.34101) 377 : cluster 0 pgmap v666: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 1.1 MiB/s wr, 37 op/s 2024-03-07T03:26:32.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:26:32 smithi195 bash[59134]: ts=2024-03-07T03:26:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:34.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:33 smithi148 bash[82222]: cluster 2024-03-07T03:26:32.519175+0000 mgr.y (mgr.34101) 378 : cluster 0 pgmap v667: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 342 KiB/s wr, 27 op/s 2024-03-07T03:26:34.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:33 smithi148 bash[79819]: cluster 2024-03-07T03:26:32.519175+0000 mgr.y (mgr.34101) 378 : cluster 0 pgmap v667: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 342 KiB/s wr, 27 op/s 2024-03-07T03:26:34.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:33 smithi195 bash[66542]: cluster 2024-03-07T03:26:32.519175+0000 mgr.y (mgr.34101) 378 : cluster 0 pgmap v667: 425 pgs: 425 active+clean; 121 MiB data, 732 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 342 KiB/s wr, 27 op/s 2024-03-07T03:26:36.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:35 smithi148 bash[79819]: cluster 2024-03-07T03:26:34.521959+0000 mgr.y (mgr.34101) 379 : cluster 0 pgmap v668: 425 pgs: 425 active+clean; 125 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 684 KiB/s wr, 31 op/s 2024-03-07T03:26:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:35 smithi148 bash[82222]: cluster 2024-03-07T03:26:34.521959+0000 mgr.y (mgr.34101) 379 : cluster 0 pgmap v668: 425 pgs: 425 active+clean; 125 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 684 KiB/s wr, 31 op/s 2024-03-07T03:26:36.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:35 smithi195 bash[66542]: cluster 2024-03-07T03:26:34.521959+0000 mgr.y (mgr.34101) 379 : cluster 0 pgmap v668: 425 pgs: 425 active+clean; 125 MiB data, 744 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 684 KiB/s wr, 31 op/s 2024-03-07T03:26:36.517 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.EncryptionLoadBadSize (5624 ms) 2024-03-07T03:26:36.517 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EncryptionLoadBadStripePattern 2024-03-07T03:26:36.517 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_librbd.cc:2840: Skipped 2024-03-07T03:26:36.517 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:26:36.517 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestLibRBD.EncryptionLoadBadStripePattern (0 ms) 2024-03-07T03:26:36.517 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EncryptionLoadFormatMismatch 2024-03-07T03:26:36.517 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:26:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:26:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:26:37] "GET /metrics HTTP/1.1" 200 60731 "" "Prometheus/2.43.0" 2024-03-07T03:26:38.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:37 smithi148 bash[82222]: cluster 2024-03-07T03:26:36.523949+0000 mgr.y (mgr.34101) 380 : cluster 0 pgmap v669: 425 pgs: 425 active+clean; 129 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 13 op/s 2024-03-07T03:26:38.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:37 smithi148 bash[79819]: cluster 2024-03-07T03:26:36.523949+0000 mgr.y (mgr.34101) 380 : cluster 0 pgmap v669: 425 pgs: 425 active+clean; 129 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 13 op/s 2024-03-07T03:26:38.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:37 smithi195 bash[66542]: cluster 2024-03-07T03:26:36.523949+0000 mgr.y (mgr.34101) 380 : cluster 0 pgmap v669: 425 pgs: 425 active+clean; 129 MiB data, 748 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 13 op/s 2024-03-07T03:26:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:39 smithi148 bash[79819]: cluster 2024-03-07T03:26:38.525038+0000 mgr.y (mgr.34101) 381 : cluster 0 pgmap v670: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 22 op/s 2024-03-07T03:26:40.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:39 smithi148 bash[82222]: cluster 2024-03-07T03:26:38.525038+0000 mgr.y (mgr.34101) 381 : cluster 0 pgmap v670: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 22 op/s 2024-03-07T03:26:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:39 smithi195 bash[66542]: cluster 2024-03-07T03:26:38.525038+0000 mgr.y (mgr.34101) 381 : cluster 0 pgmap v670: 425 pgs: 425 active+clean; 129 MiB data, 752 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 683 KiB/s wr, 22 op/s 2024-03-07T03:26:42.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:41 smithi148 bash[79819]: cluster 2024-03-07T03:26:40.527338+0000 mgr.y (mgr.34101) 382 : cluster 0 pgmap v671: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 491 KiB/s rd, 683 KiB/s wr, 23 op/s 2024-03-07T03:26:42.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:41 smithi148 bash[79819]: cluster 2024-03-07T03:26:40.613481+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2024-03-07T03:26:42.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:41 smithi148 bash[82222]: cluster 2024-03-07T03:26:40.527338+0000 mgr.y (mgr.34101) 382 : cluster 0 pgmap v671: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 491 KiB/s rd, 683 KiB/s wr, 23 op/s 2024-03-07T03:26:42.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:41 smithi148 bash[82222]: cluster 2024-03-07T03:26:40.613481+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2024-03-07T03:26:42.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:41 smithi195 bash[66542]: cluster 2024-03-07T03:26:40.527338+0000 mgr.y (mgr.34101) 382 : cluster 0 pgmap v671: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 491 KiB/s rd, 683 KiB/s wr, 23 op/s 2024-03-07T03:26:42.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:41 smithi195 bash[66542]: cluster 2024-03-07T03:26:40.613481+0000 mon.a (mon.0) 1099 : cluster 0 osdmap e671: 8 total, 8 up, 8 in 2024-03-07T03:26:42.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:26:42 smithi195 bash[59134]: ts=2024-03-07T03:26:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:42 smithi148 bash[79819]: cluster 2024-03-07T03:26:41.623693+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2024-03-07T03:26:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:42 smithi148 bash[79819]: audit 2024-03-07T03:26:42.318379+0000 mon.a (mon.0) 1101 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:43.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:42 smithi148 bash[82222]: cluster 2024-03-07T03:26:41.623693+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2024-03-07T03:26:43.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:42 smithi148 bash[82222]: audit 2024-03-07T03:26:42.318379+0000 mon.a (mon.0) 1101 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:42 smithi195 bash[66542]: cluster 2024-03-07T03:26:41.623693+0000 mon.a (mon.0) 1100 : cluster 0 osdmap e672: 8 total, 8 up, 8 in 2024-03-07T03:26:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:42 smithi195 bash[66542]: audit 2024-03-07T03:26:42.318379+0000 mon.a (mon.0) 1101 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:44.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:43 smithi148 bash[79819]: cluster 2024-03-07T03:26:42.527863+0000 mgr.y (mgr.34101) 383 : cluster 0 pgmap v674: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 513 KiB/s wr, 23 op/s 2024-03-07T03:26:44.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:43 smithi148 bash[82222]: cluster 2024-03-07T03:26:42.527863+0000 mgr.y (mgr.34101) 383 : cluster 0 pgmap v674: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 513 KiB/s wr, 23 op/s 2024-03-07T03:26:44.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:43 smithi195 bash[66542]: cluster 2024-03-07T03:26:42.527863+0000 mgr.y (mgr.34101) 383 : cluster 0 pgmap v674: 425 pgs: 425 active+clean; 129 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 513 KiB/s wr, 23 op/s 2024-03-07T03:26:46.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:45 smithi148 bash[79819]: cluster 2024-03-07T03:26:44.529883+0000 mgr.y (mgr.34101) 384 : cluster 0 pgmap v675: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 515 KiB/s wr, 82 op/s 2024-03-07T03:26:46.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:45 smithi148 bash[82222]: cluster 2024-03-07T03:26:44.529883+0000 mgr.y (mgr.34101) 384 : cluster 0 pgmap v675: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 515 KiB/s wr, 82 op/s 2024-03-07T03:26:46.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:45 smithi195 bash[66542]: cluster 2024-03-07T03:26:44.529883+0000 mgr.y (mgr.34101) 384 : cluster 0 pgmap v675: 425 pgs: 425 active+clean; 133 MiB data, 764 MiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 515 KiB/s wr, 82 op/s 2024-03-07T03:26:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:26:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:26:47] "GET /metrics HTTP/1.1" 200 60729 "" "Prometheus/2.43.0" 2024-03-07T03:26:48.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:47 smithi148 bash[79819]: cluster 2024-03-07T03:26:46.531230+0000 mgr.y (mgr.34101) 385 : cluster 0 pgmap v676: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 515 KiB/s wr, 73 op/s 2024-03-07T03:26:48.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:47 smithi148 bash[82222]: cluster 2024-03-07T03:26:46.531230+0000 mgr.y (mgr.34101) 385 : cluster 0 pgmap v676: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 515 KiB/s wr, 73 op/s 2024-03-07T03:26:48.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:47 smithi195 bash[66542]: cluster 2024-03-07T03:26:46.531230+0000 mgr.y (mgr.34101) 385 : cluster 0 pgmap v676: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 92 KiB/s rd, 515 KiB/s wr, 73 op/s 2024-03-07T03:26:50.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:49 smithi148 bash[79819]: cluster 2024-03-07T03:26:48.532597+0000 mgr.y (mgr.34101) 386 : cluster 0 pgmap v677: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 515 KiB/s wr, 68 op/s 2024-03-07T03:26:50.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:49 smithi148 bash[82222]: cluster 2024-03-07T03:26:48.532597+0000 mgr.y (mgr.34101) 386 : cluster 0 pgmap v677: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 515 KiB/s wr, 68 op/s 2024-03-07T03:26:50.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:49 smithi195 bash[66542]: cluster 2024-03-07T03:26:48.532597+0000 mgr.y (mgr.34101) 386 : cluster 0 pgmap v677: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 515 KiB/s wr, 68 op/s 2024-03-07T03:26:52.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:51 smithi148 bash[79819]: cluster 2024-03-07T03:26:50.534162+0000 mgr.y (mgr.34101) 387 : cluster 0 pgmap v678: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 415 KiB/s wr, 55 op/s 2024-03-07T03:26:52.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:51 smithi148 bash[82222]: cluster 2024-03-07T03:26:50.534162+0000 mgr.y (mgr.34101) 387 : cluster 0 pgmap v678: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 415 KiB/s wr, 55 op/s 2024-03-07T03:26:52.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:51 smithi195 bash[66542]: cluster 2024-03-07T03:26:50.534162+0000 mgr.y (mgr.34101) 387 : cluster 0 pgmap v678: 425 pgs: 425 active+clean; 133 MiB data, 768 MiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 415 KiB/s wr, 55 op/s 2024-03-07T03:26:52.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:26:52 smithi195 bash[59134]: ts=2024-03-07T03:26:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:26:54.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:53 smithi148 bash[79819]: cluster 2024-03-07T03:26:52.535120+0000 mgr.y (mgr.34101) 388 : cluster 0 pgmap v679: 425 pgs: 425 active+clean; 133 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 377 KiB/s wr, 49 op/s 2024-03-07T03:26:54.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:53 smithi148 bash[82222]: cluster 2024-03-07T03:26:52.535120+0000 mgr.y (mgr.34101) 388 : cluster 0 pgmap v679: 425 pgs: 425 active+clean; 133 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 377 KiB/s wr, 49 op/s 2024-03-07T03:26:54.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:53 smithi195 bash[66542]: cluster 2024-03-07T03:26:52.535120+0000 mgr.y (mgr.34101) 388 : cluster 0 pgmap v679: 425 pgs: 425 active+clean; 133 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 377 KiB/s wr, 49 op/s 2024-03-07T03:26:55.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:54 smithi148 bash[79819]: cluster 2024-03-07T03:26:54.536835+0000 mgr.y (mgr.34101) 389 : cluster 0 pgmap v680: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.7 MiB/s wr, 57 op/s 2024-03-07T03:26:55.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:54 smithi148 bash[82222]: cluster 2024-03-07T03:26:54.536835+0000 mgr.y (mgr.34101) 389 : cluster 0 pgmap v680: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.7 MiB/s wr, 57 op/s 2024-03-07T03:26:55.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:54 smithi195 bash[66542]: cluster 2024-03-07T03:26:54.536835+0000 mgr.y (mgr.34101) 389 : cluster 0 pgmap v680: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1.7 MiB/s wr, 57 op/s 2024-03-07T03:26:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:26:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:26:57] "GET /metrics HTTP/1.1" 200 60729 "" "Prometheus/2.43.0" 2024-03-07T03:26:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:57 smithi148 bash[79819]: cluster 2024-03-07T03:26:56.538174+0000 mgr.y (mgr.34101) 390 : cluster 0 pgmap v681: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-03-07T03:26:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:57 smithi148 bash[79819]: audit 2024-03-07T03:26:57.318608+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:57 smithi148 bash[82222]: cluster 2024-03-07T03:26:56.538174+0000 mgr.y (mgr.34101) 390 : cluster 0 pgmap v681: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-03-07T03:26:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:57 smithi148 bash[82222]: audit 2024-03-07T03:26:57.318608+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:26:58.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:57 smithi195 bash[66542]: cluster 2024-03-07T03:26:56.538174+0000 mgr.y (mgr.34101) 390 : cluster 0 pgmap v681: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-03-07T03:26:58.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:57 smithi195 bash[66542]: audit 2024-03-07T03:26:57.318608+0000 mon.a (mon.0) 1102 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:00.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:26:59 smithi148 bash[79819]: cluster 2024-03-07T03:26:58.540084+0000 mgr.y (mgr.34101) 391 : cluster 0 pgmap v682: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:00.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:26:59 smithi148 bash[82222]: cluster 2024-03-07T03:26:58.540084+0000 mgr.y (mgr.34101) 391 : cluster 0 pgmap v682: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:00.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:26:59 smithi195 bash[66542]: cluster 2024-03-07T03:26:58.540084+0000 mgr.y (mgr.34101) 391 : cluster 0 pgmap v682: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:02.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:01 smithi148 bash[79819]: cluster 2024-03-07T03:27:00.541599+0000 mgr.y (mgr.34101) 392 : cluster 0 pgmap v683: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:02.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:01 smithi148 bash[82222]: cluster 2024-03-07T03:27:00.541599+0000 mgr.y (mgr.34101) 392 : cluster 0 pgmap v683: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:02.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:01 smithi195 bash[66542]: cluster 2024-03-07T03:27:00.541599+0000 mgr.y (mgr.34101) 392 : cluster 0 pgmap v683: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:02.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:27:02 smithi195 bash[59134]: ts=2024-03-07T03:27:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:04.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:03 smithi148 bash[79819]: cluster 2024-03-07T03:27:02.542574+0000 mgr.y (mgr.34101) 393 : cluster 0 pgmap v684: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:04.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:03 smithi148 bash[82222]: cluster 2024-03-07T03:27:02.542574+0000 mgr.y (mgr.34101) 393 : cluster 0 pgmap v684: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:04.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:03 smithi195 bash[66542]: cluster 2024-03-07T03:27:02.542574+0000 mgr.y (mgr.34101) 393 : cluster 0 pgmap v684: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-03-07T03:27:06.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:05 smithi148 bash[79819]: cluster 2024-03-07T03:27:04.545058+0000 mgr.y (mgr.34101) 394 : cluster 0 pgmap v685: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-03-07T03:27:06.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:05 smithi148 bash[82222]: cluster 2024-03-07T03:27:04.545058+0000 mgr.y (mgr.34101) 394 : cluster 0 pgmap v685: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-03-07T03:27:06.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:05 smithi195 bash[66542]: cluster 2024-03-07T03:27:04.545058+0000 mgr.y (mgr.34101) 394 : cluster 0 pgmap v685: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 1.3 MiB/s wr, 16 op/s 2024-03-07T03:27:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:27:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:27:07] "GET /metrics HTTP/1.1" 200 60733 "" "Prometheus/2.43.0" 2024-03-07T03:27:08.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:07 smithi148 bash[79819]: cluster 2024-03-07T03:27:06.547002+0000 mgr.y (mgr.34101) 395 : cluster 0 pgmap v686: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 2 op/s 2024-03-07T03:27:08.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:07 smithi148 bash[82222]: cluster 2024-03-07T03:27:06.547002+0000 mgr.y (mgr.34101) 395 : cluster 0 pgmap v686: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 2 op/s 2024-03-07T03:27:08.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:07 smithi195 bash[66542]: cluster 2024-03-07T03:27:06.547002+0000 mgr.y (mgr.34101) 395 : cluster 0 pgmap v686: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 2 op/s 2024-03-07T03:27:10.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:09 smithi148 bash[79819]: cluster 2024-03-07T03:27:08.548133+0000 mgr.y (mgr.34101) 396 : cluster 0 pgmap v687: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 3 op/s 2024-03-07T03:27:10.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:09 smithi148 bash[82222]: cluster 2024-03-07T03:27:08.548133+0000 mgr.y (mgr.34101) 396 : cluster 0 pgmap v687: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 3 op/s 2024-03-07T03:27:10.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:09 smithi195 bash[66542]: cluster 2024-03-07T03:27:08.548133+0000 mgr.y (mgr.34101) 396 : cluster 0 pgmap v687: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 3 op/s 2024-03-07T03:27:12.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:11 smithi148 bash[79819]: cluster 2024-03-07T03:27:10.549398+0000 mgr.y (mgr.34101) 397 : cluster 0 pgmap v688: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:12.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:11 smithi148 bash[82222]: cluster 2024-03-07T03:27:10.549398+0000 mgr.y (mgr.34101) 397 : cluster 0 pgmap v688: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:12.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:11 smithi195 bash[66542]: cluster 2024-03-07T03:27:10.549398+0000 mgr.y (mgr.34101) 397 : cluster 0 pgmap v688: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:12.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:27:12 smithi195 bash[59134]: ts=2024-03-07T03:27:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:12 smithi148 bash[79819]: audit 2024-03-07T03:27:12.318927+0000 mon.a (mon.0) 1103 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:13.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:12 smithi148 bash[82222]: audit 2024-03-07T03:27:12.318927+0000 mon.a (mon.0) 1103 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:13.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:12 smithi195 bash[66542]: audit 2024-03-07T03:27:12.318927+0000 mon.a (mon.0) 1103 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:14.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:13 smithi148 bash[79819]: cluster 2024-03-07T03:27:12.550327+0000 mgr.y (mgr.34101) 398 : cluster 0 pgmap v689: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:14.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:13 smithi148 bash[82222]: cluster 2024-03-07T03:27:12.550327+0000 mgr.y (mgr.34101) 398 : cluster 0 pgmap v689: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:14.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:13 smithi195 bash[66542]: cluster 2024-03-07T03:27:12.550327+0000 mgr.y (mgr.34101) 398 : cluster 0 pgmap v689: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:16.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:15 smithi148 bash[79819]: cluster 2024-03-07T03:27:14.551634+0000 mgr.y (mgr.34101) 399 : cluster 0 pgmap v690: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:16.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:15 smithi148 bash[82222]: cluster 2024-03-07T03:27:14.551634+0000 mgr.y (mgr.34101) 399 : cluster 0 pgmap v690: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:16.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:15 smithi195 bash[66542]: cluster 2024-03-07T03:27:14.551634+0000 mgr.y (mgr.34101) 399 : cluster 0 pgmap v690: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:27:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:27:17] "GET /metrics HTTP/1.1" 200 60732 "" "Prometheus/2.43.0" 2024-03-07T03:27:18.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:17 smithi148 bash[79819]: cluster 2024-03-07T03:27:16.553223+0000 mgr.y (mgr.34101) 400 : cluster 0 pgmap v691: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-03-07T03:27:18.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:17 smithi148 bash[82222]: cluster 2024-03-07T03:27:16.553223+0000 mgr.y (mgr.34101) 400 : cluster 0 pgmap v691: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-03-07T03:27:18.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:17 smithi195 bash[66542]: cluster 2024-03-07T03:27:16.553223+0000 mgr.y (mgr.34101) 400 : cluster 0 pgmap v691: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-03-07T03:27:20.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:19 smithi148 bash[82222]: cluster 2024-03-07T03:27:18.554535+0000 mgr.y (mgr.34101) 401 : cluster 0 pgmap v692: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:20.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:19 smithi148 bash[79819]: cluster 2024-03-07T03:27:18.554535+0000 mgr.y (mgr.34101) 401 : cluster 0 pgmap v692: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:20.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:19 smithi195 bash[66542]: cluster 2024-03-07T03:27:18.554535+0000 mgr.y (mgr.34101) 401 : cluster 0 pgmap v692: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:22.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:21 smithi148 bash[79819]: cluster 2024-03-07T03:27:20.556816+0000 mgr.y (mgr.34101) 402 : cluster 0 pgmap v693: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:27:22.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:21 smithi148 bash[79819]: audit 2024-03-07T03:27:21.325918+0000 mon.a (mon.0) 1104 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:27:22.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:21 smithi148 bash[82222]: cluster 2024-03-07T03:27:20.556816+0000 mgr.y (mgr.34101) 402 : cluster 0 pgmap v693: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:27:22.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:21 smithi148 bash[82222]: audit 2024-03-07T03:27:21.325918+0000 mon.a (mon.0) 1104 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:27:22.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:21 smithi195 bash[66542]: cluster 2024-03-07T03:27:20.556816+0000 mgr.y (mgr.34101) 402 : cluster 0 pgmap v693: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:27:22.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:21 smithi195 bash[66542]: audit 2024-03-07T03:27:21.325918+0000 mon.a (mon.0) 1104 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:27:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:27:22 smithi195 bash[59134]: ts=2024-03-07T03:27:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:24.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:23 smithi148 bash[79819]: cluster 2024-03-07T03:27:22.557807+0000 mgr.y (mgr.34101) 403 : cluster 0 pgmap v694: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:27:24.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:23 smithi148 bash[82222]: cluster 2024-03-07T03:27:22.557807+0000 mgr.y (mgr.34101) 403 : cluster 0 pgmap v694: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:27:24.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:23 smithi195 bash[66542]: cluster 2024-03-07T03:27:22.557807+0000 mgr.y (mgr.34101) 403 : cluster 0 pgmap v694: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-03-07T03:27:26.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:25 smithi148 bash[82222]: cluster 2024-03-07T03:27:24.561704+0000 mgr.y (mgr.34101) 404 : cluster 0 pgmap v695: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:26.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:25 smithi148 bash[79819]: cluster 2024-03-07T03:27:24.561704+0000 mgr.y (mgr.34101) 404 : cluster 0 pgmap v695: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:26.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:25 smithi195 bash[66542]: cluster 2024-03-07T03:27:24.561704+0000 mgr.y (mgr.34101) 404 : cluster 0 pgmap v695: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-03-07T03:27:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:27:27] "GET /metrics HTTP/1.1" 200 60732 "" "Prometheus/2.43.0" 2024-03-07T03:27:28.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[79819]: cluster 2024-03-07T03:27:26.563037+0000 mgr.y (mgr.34101) 405 : cluster 0 pgmap v696: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[79819]: audit 2024-03-07T03:27:27.062398+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[79819]: audit 2024-03-07T03:27:27.069472+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[79819]: audit 2024-03-07T03:27:27.319519+0000 mon.a (mon.0) 1107 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[79819]: audit 2024-03-07T03:27:27.457740+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[79819]: audit 2024-03-07T03:27:27.463592+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[82222]: cluster 2024-03-07T03:27:26.563037+0000 mgr.y (mgr.34101) 405 : cluster 0 pgmap v696: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[82222]: audit 2024-03-07T03:27:27.062398+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[82222]: audit 2024-03-07T03:27:27.069472+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[82222]: audit 2024-03-07T03:27:27.319519+0000 mon.a (mon.0) 1107 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:28.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[82222]: audit 2024-03-07T03:27:27.457740+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:27 smithi148 bash[82222]: audit 2024-03-07T03:27:27.463592+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:27 smithi195 bash[66542]: cluster 2024-03-07T03:27:26.563037+0000 mgr.y (mgr.34101) 405 : cluster 0 pgmap v696: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-03-07T03:27:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:27 smithi195 bash[66542]: audit 2024-03-07T03:27:27.062398+0000 mon.a (mon.0) 1105 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:27 smithi195 bash[66542]: audit 2024-03-07T03:27:27.069472+0000 mon.a (mon.0) 1106 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:27 smithi195 bash[66542]: audit 2024-03-07T03:27:27.319519+0000 mon.a (mon.0) 1107 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:27 smithi195 bash[66542]: audit 2024-03-07T03:27:27.457740+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:27 smithi195 bash[66542]: audit 2024-03-07T03:27:27.463592+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:29.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[79819]: audit 2024-03-07T03:27:28.049871+0000 mon.a (mon.0) 1110 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:27:29.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[79819]: audit 2024-03-07T03:27:28.051289+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:27:29.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[79819]: audit 2024-03-07T03:27:28.057751+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:29.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[79819]: cluster 2024-03-07T03:27:28.564390+0000 mgr.y (mgr.34101) 406 : cluster 0 pgmap v697: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 4 op/s 2024-03-07T03:27:29.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[82222]: audit 2024-03-07T03:27:28.049871+0000 mon.a (mon.0) 1110 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:27:29.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[82222]: audit 2024-03-07T03:27:28.051289+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:27:29.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[82222]: audit 2024-03-07T03:27:28.057751+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:29.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:28 smithi148 bash[82222]: cluster 2024-03-07T03:27:28.564390+0000 mgr.y (mgr.34101) 406 : cluster 0 pgmap v697: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 4 op/s 2024-03-07T03:27:29.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:28 smithi195 bash[66542]: audit 2024-03-07T03:27:28.049871+0000 mon.a (mon.0) 1110 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:27:29.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:28 smithi195 bash[66542]: audit 2024-03-07T03:27:28.051289+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:27:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:28 smithi195 bash[66542]: audit 2024-03-07T03:27:28.057751+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:27:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:28 smithi195 bash[66542]: cluster 2024-03-07T03:27:28.564390+0000 mgr.y (mgr.34101) 406 : cluster 0 pgmap v697: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 4 op/s 2024-03-07T03:27:32.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:31 smithi148 bash[79819]: cluster 2024-03-07T03:27:30.566628+0000 mgr.y (mgr.34101) 407 : cluster 0 pgmap v698: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 8 op/s 2024-03-07T03:27:32.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:31 smithi148 bash[82222]: cluster 2024-03-07T03:27:30.566628+0000 mgr.y (mgr.34101) 407 : cluster 0 pgmap v698: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 8 op/s 2024-03-07T03:27:32.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:31 smithi195 bash[66542]: cluster 2024-03-07T03:27:30.566628+0000 mgr.y (mgr.34101) 407 : cluster 0 pgmap v698: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 8 op/s 2024-03-07T03:27:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:27:32 smithi195 bash[59134]: ts=2024-03-07T03:27:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:33 smithi148 bash[79819]: cluster 2024-03-07T03:27:32.568186+0000 mgr.y (mgr.34101) 408 : cluster 0 pgmap v699: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 8 op/s 2024-03-07T03:27:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:33 smithi148 bash[82222]: cluster 2024-03-07T03:27:32.568186+0000 mgr.y (mgr.34101) 408 : cluster 0 pgmap v699: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 8 op/s 2024-03-07T03:27:34.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:33 smithi195 bash[66542]: cluster 2024-03-07T03:27:32.568186+0000 mgr.y (mgr.34101) 408 : cluster 0 pgmap v699: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 8 op/s 2024-03-07T03:27:36.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:35 smithi148 bash[79819]: cluster 2024-03-07T03:27:34.570045+0000 mgr.y (mgr.34101) 409 : cluster 0 pgmap v700: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 516 KiB/s rd, 8 op/s 2024-03-07T03:27:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:35 smithi148 bash[82222]: cluster 2024-03-07T03:27:34.570045+0000 mgr.y (mgr.34101) 409 : cluster 0 pgmap v700: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 516 KiB/s rd, 8 op/s 2024-03-07T03:27:36.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:35 smithi195 bash[66542]: cluster 2024-03-07T03:27:34.570045+0000 mgr.y (mgr.34101) 409 : cluster 0 pgmap v700: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 516 KiB/s rd, 8 op/s 2024-03-07T03:27:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:27:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:27:37] "GET /metrics HTTP/1.1" 200 60729 "" "Prometheus/2.43.0" 2024-03-07T03:27:38.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:37 smithi148 bash[79819]: cluster 2024-03-07T03:27:36.571094+0000 mgr.y (mgr.34101) 410 : cluster 0 pgmap v701: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:27:38.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:37 smithi148 bash[82222]: cluster 2024-03-07T03:27:36.571094+0000 mgr.y (mgr.34101) 410 : cluster 0 pgmap v701: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:27:38.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:37 smithi195 bash[66542]: cluster 2024-03-07T03:27:36.571094+0000 mgr.y (mgr.34101) 410 : cluster 0 pgmap v701: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:27:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:39 smithi148 bash[79819]: cluster 2024-03-07T03:27:38.573010+0000 mgr.y (mgr.34101) 411 : cluster 0 pgmap v702: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-03-07T03:27:40.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:39 smithi148 bash[82222]: cluster 2024-03-07T03:27:38.573010+0000 mgr.y (mgr.34101) 411 : cluster 0 pgmap v702: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-03-07T03:27:40.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:39 smithi195 bash[66542]: cluster 2024-03-07T03:27:38.573010+0000 mgr.y (mgr.34101) 411 : cluster 0 pgmap v702: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-03-07T03:27:42.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:41 smithi148 bash[79819]: cluster 2024-03-07T03:27:40.574564+0000 mgr.y (mgr.34101) 412 : cluster 0 pgmap v703: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 85 B/s wr, 13 op/s 2024-03-07T03:27:42.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:41 smithi148 bash[82222]: cluster 2024-03-07T03:27:40.574564+0000 mgr.y (mgr.34101) 412 : cluster 0 pgmap v703: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 85 B/s wr, 13 op/s 2024-03-07T03:27:42.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:41 smithi195 bash[66542]: cluster 2024-03-07T03:27:40.574564+0000 mgr.y (mgr.34101) 412 : cluster 0 pgmap v703: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 85 B/s wr, 13 op/s 2024-03-07T03:27:42.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:27:42 smithi195 bash[59134]: ts=2024-03-07T03:27:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:43 smithi148 bash[79819]: audit 2024-03-07T03:27:42.320065+0000 mon.a (mon.0) 1113 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:43.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:43 smithi148 bash[79819]: cluster 2024-03-07T03:27:42.575326+0000 mgr.y (mgr.34101) 413 : cluster 0 pgmap v704: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 85 B/s wr, 8 op/s 2024-03-07T03:27:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:43 smithi148 bash[82222]: audit 2024-03-07T03:27:42.320065+0000 mon.a (mon.0) 1113 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:43 smithi148 bash[82222]: cluster 2024-03-07T03:27:42.575326+0000 mgr.y (mgr.34101) 413 : cluster 0 pgmap v704: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 85 B/s wr, 8 op/s 2024-03-07T03:27:43.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:43 smithi195 bash[66542]: audit 2024-03-07T03:27:42.320065+0000 mon.a (mon.0) 1113 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:43.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:43 smithi195 bash[66542]: cluster 2024-03-07T03:27:42.575326+0000 mgr.y (mgr.34101) 413 : cluster 0 pgmap v704: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 85 B/s wr, 8 op/s 2024-03-07T03:27:46.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:45 smithi148 bash[79819]: cluster 2024-03-07T03:27:44.578045+0000 mgr.y (mgr.34101) 414 : cluster 0 pgmap v705: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:46.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:45 smithi148 bash[82222]: cluster 2024-03-07T03:27:44.578045+0000 mgr.y (mgr.34101) 414 : cluster 0 pgmap v705: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:46.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:45 smithi195 bash[66542]: cluster 2024-03-07T03:27:44.578045+0000 mgr.y (mgr.34101) 414 : cluster 0 pgmap v705: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:27:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:27:47] "GET /metrics HTTP/1.1" 200 60732 "" "Prometheus/2.43.0" 2024-03-07T03:27:48.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:47 smithi148 bash[79819]: cluster 2024-03-07T03:27:46.579338+0000 mgr.y (mgr.34101) 415 : cluster 0 pgmap v706: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:48.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:47 smithi148 bash[82222]: cluster 2024-03-07T03:27:46.579338+0000 mgr.y (mgr.34101) 415 : cluster 0 pgmap v706: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:48.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:47 smithi195 bash[66542]: cluster 2024-03-07T03:27:46.579338+0000 mgr.y (mgr.34101) 415 : cluster 0 pgmap v706: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:50.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:49 smithi148 bash[79819]: cluster 2024-03-07T03:27:48.581861+0000 mgr.y (mgr.34101) 416 : cluster 0 pgmap v707: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 170 B/s wr, 19 op/s 2024-03-07T03:27:50.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:49 smithi148 bash[82222]: cluster 2024-03-07T03:27:48.581861+0000 mgr.y (mgr.34101) 416 : cluster 0 pgmap v707: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 170 B/s wr, 19 op/s 2024-03-07T03:27:50.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:49 smithi195 bash[66542]: cluster 2024-03-07T03:27:48.581861+0000 mgr.y (mgr.34101) 416 : cluster 0 pgmap v707: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 170 B/s wr, 19 op/s 2024-03-07T03:27:52.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:51 smithi148 bash[82222]: cluster 2024-03-07T03:27:50.584215+0000 mgr.y (mgr.34101) 417 : cluster 0 pgmap v708: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 255 B/s wr, 21 op/s 2024-03-07T03:27:52.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:51 smithi148 bash[79819]: cluster 2024-03-07T03:27:50.584215+0000 mgr.y (mgr.34101) 417 : cluster 0 pgmap v708: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 255 B/s wr, 21 op/s 2024-03-07T03:27:52.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:51 smithi195 bash[66542]: cluster 2024-03-07T03:27:50.584215+0000 mgr.y (mgr.34101) 417 : cluster 0 pgmap v708: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 255 B/s wr, 21 op/s 2024-03-07T03:27:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:27:52 smithi195 bash[59134]: ts=2024-03-07T03:27:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:27:54.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:53 smithi148 bash[79819]: cluster 2024-03-07T03:27:52.585205+0000 mgr.y (mgr.34101) 418 : cluster 0 pgmap v709: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:54.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:53 smithi148 bash[82222]: cluster 2024-03-07T03:27:52.585205+0000 mgr.y (mgr.34101) 418 : cluster 0 pgmap v709: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:54.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:53 smithi195 bash[66542]: cluster 2024-03-07T03:27:52.585205+0000 mgr.y (mgr.34101) 418 : cluster 0 pgmap v709: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-03-07T03:27:56.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:55 smithi148 bash[82222]: cluster 2024-03-07T03:27:54.587876+0000 mgr.y (mgr.34101) 419 : cluster 0 pgmap v710: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 170 B/s wr, 24 op/s 2024-03-07T03:27:56.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:55 smithi148 bash[79819]: cluster 2024-03-07T03:27:54.587876+0000 mgr.y (mgr.34101) 419 : cluster 0 pgmap v710: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 170 B/s wr, 24 op/s 2024-03-07T03:27:56.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:55 smithi195 bash[66542]: cluster 2024-03-07T03:27:54.587876+0000 mgr.y (mgr.34101) 419 : cluster 0 pgmap v710: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 170 B/s wr, 24 op/s 2024-03-07T03:27:57.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:27:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:27:57] "GET /metrics HTTP/1.1" 200 60732 "" "Prometheus/2.43.0" 2024-03-07T03:27:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:57 smithi148 bash[82222]: cluster 2024-03-07T03:27:56.591289+0000 mgr.y (mgr.34101) 420 : cluster 0 pgmap v711: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:27:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:57 smithi148 bash[82222]: audit 2024-03-07T03:27:57.319222+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:57 smithi148 bash[79819]: cluster 2024-03-07T03:27:56.591289+0000 mgr.y (mgr.34101) 420 : cluster 0 pgmap v711: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:27:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:57 smithi148 bash[79819]: audit 2024-03-07T03:27:57.319222+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:27:58.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:57 smithi195 bash[66542]: cluster 2024-03-07T03:27:56.591289+0000 mgr.y (mgr.34101) 420 : cluster 0 pgmap v711: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:27:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:57 smithi195 bash[66542]: audit 2024-03-07T03:27:57.319222+0000 mon.a (mon.0) 1114 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:00.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:27:59 smithi148 bash[82222]: cluster 2024-03-07T03:27:58.592674+0000 mgr.y (mgr.34101) 421 : cluster 0 pgmap v712: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 85 B/s wr, 19 op/s 2024-03-07T03:28:00.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:27:59 smithi148 bash[79819]: cluster 2024-03-07T03:27:58.592674+0000 mgr.y (mgr.34101) 421 : cluster 0 pgmap v712: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 85 B/s wr, 19 op/s 2024-03-07T03:28:00.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:27:59 smithi195 bash[66542]: cluster 2024-03-07T03:27:58.592674+0000 mgr.y (mgr.34101) 421 : cluster 0 pgmap v712: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 85 B/s wr, 19 op/s 2024-03-07T03:28:01.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:00 smithi148 bash[82222]: cluster 2024-03-07T03:28:00.594996+0000 mgr.y (mgr.34101) 422 : cluster 0 pgmap v713: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 85 B/s wr, 21 op/s 2024-03-07T03:28:01.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:00 smithi148 bash[79819]: cluster 2024-03-07T03:28:00.594996+0000 mgr.y (mgr.34101) 422 : cluster 0 pgmap v713: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 85 B/s wr, 21 op/s 2024-03-07T03:28:01.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:00 smithi195 bash[66542]: cluster 2024-03-07T03:28:00.594996+0000 mgr.y (mgr.34101) 422 : cluster 0 pgmap v713: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 85 B/s wr, 21 op/s 2024-03-07T03:28:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:28:02 smithi195 bash[59134]: ts=2024-03-07T03:28:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:04.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:03 smithi148 bash[82222]: cluster 2024-03-07T03:28:02.596375+0000 mgr.y (mgr.34101) 423 : cluster 0 pgmap v714: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:04.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:03 smithi148 bash[79819]: cluster 2024-03-07T03:28:02.596375+0000 mgr.y (mgr.34101) 423 : cluster 0 pgmap v714: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:04.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:03 smithi195 bash[66542]: cluster 2024-03-07T03:28:02.596375+0000 mgr.y (mgr.34101) 423 : cluster 0 pgmap v714: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:06.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:05 smithi148 bash[79819]: cluster 2024-03-07T03:28:04.598136+0000 mgr.y (mgr.34101) 424 : cluster 0 pgmap v715: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 16 op/s 2024-03-07T03:28:06.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:05 smithi148 bash[82222]: cluster 2024-03-07T03:28:04.598136+0000 mgr.y (mgr.34101) 424 : cluster 0 pgmap v715: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 16 op/s 2024-03-07T03:28:06.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:05 smithi195 bash[66542]: cluster 2024-03-07T03:28:04.598136+0000 mgr.y (mgr.34101) 424 : cluster 0 pgmap v715: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 16 op/s 2024-03-07T03:28:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:28:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:28:07] "GET /metrics HTTP/1.1" 200 60733 "" "Prometheus/2.43.0" 2024-03-07T03:28:08.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:07 smithi148 bash[79819]: cluster 2024-03-07T03:28:06.600083+0000 mgr.y (mgr.34101) 425 : cluster 0 pgmap v716: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:08.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:07 smithi148 bash[82222]: cluster 2024-03-07T03:28:06.600083+0000 mgr.y (mgr.34101) 425 : cluster 0 pgmap v716: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:08.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:07 smithi195 bash[66542]: cluster 2024-03-07T03:28:06.600083+0000 mgr.y (mgr.34101) 425 : cluster 0 pgmap v716: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:10.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:09 smithi148 bash[82222]: cluster 2024-03-07T03:28:08.602117+0000 mgr.y (mgr.34101) 426 : cluster 0 pgmap v717: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-03-07T03:28:10.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:09 smithi148 bash[79819]: cluster 2024-03-07T03:28:08.602117+0000 mgr.y (mgr.34101) 426 : cluster 0 pgmap v717: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-03-07T03:28:10.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:09 smithi195 bash[66542]: cluster 2024-03-07T03:28:08.602117+0000 mgr.y (mgr.34101) 426 : cluster 0 pgmap v717: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-03-07T03:28:11.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:10 smithi148 bash[79819]: cluster 2024-03-07T03:28:10.603711+0000 mgr.y (mgr.34101) 427 : cluster 0 pgmap v718: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 13 op/s 2024-03-07T03:28:11.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:10 smithi148 bash[82222]: cluster 2024-03-07T03:28:10.603711+0000 mgr.y (mgr.34101) 427 : cluster 0 pgmap v718: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 13 op/s 2024-03-07T03:28:11.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:10 smithi195 bash[66542]: cluster 2024-03-07T03:28:10.603711+0000 mgr.y (mgr.34101) 427 : cluster 0 pgmap v718: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 13 op/s 2024-03-07T03:28:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:12 smithi195 bash[66542]: audit 2024-03-07T03:28:12.320554+0000 mon.a (mon.0) 1115 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:28:12 smithi195 bash[59134]: ts=2024-03-07T03:28:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:12 smithi148 bash[79819]: audit 2024-03-07T03:28:12.320554+0000 mon.a (mon.0) 1115 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:12 smithi148 bash[82222]: audit 2024-03-07T03:28:12.320554+0000 mon.a (mon.0) 1115 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:13 smithi148 bash[82222]: cluster 2024-03-07T03:28:12.604626+0000 mgr.y (mgr.34101) 428 : cluster 0 pgmap v719: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:13 smithi148 bash[79819]: cluster 2024-03-07T03:28:12.604626+0000 mgr.y (mgr.34101) 428 : cluster 0 pgmap v719: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:13.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:13 smithi195 bash[66542]: cluster 2024-03-07T03:28:12.604626+0000 mgr.y (mgr.34101) 428 : cluster 0 pgmap v719: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:16.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:15 smithi148 bash[79819]: cluster 2024-03-07T03:28:14.606077+0000 mgr.y (mgr.34101) 429 : cluster 0 pgmap v720: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 16 op/s 2024-03-07T03:28:16.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:15 smithi148 bash[82222]: cluster 2024-03-07T03:28:14.606077+0000 mgr.y (mgr.34101) 429 : cluster 0 pgmap v720: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 16 op/s 2024-03-07T03:28:16.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:15 smithi195 bash[66542]: cluster 2024-03-07T03:28:14.606077+0000 mgr.y (mgr.34101) 429 : cluster 0 pgmap v720: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 16 op/s 2024-03-07T03:28:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:28:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:28:17] "GET /metrics HTTP/1.1" 200 60732 "" "Prometheus/2.43.0" 2024-03-07T03:28:18.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:17 smithi148 bash[79819]: cluster 2024-03-07T03:28:16.607329+0000 mgr.y (mgr.34101) 430 : cluster 0 pgmap v721: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:18.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:17 smithi148 bash[82222]: cluster 2024-03-07T03:28:16.607329+0000 mgr.y (mgr.34101) 430 : cluster 0 pgmap v721: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:18.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:17 smithi195 bash[66542]: cluster 2024-03-07T03:28:16.607329+0000 mgr.y (mgr.34101) 430 : cluster 0 pgmap v721: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:19.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:18 smithi148 bash[79819]: cluster 2024-03-07T03:28:18.609244+0000 mgr.y (mgr.34101) 431 : cluster 0 pgmap v722: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 19 op/s 2024-03-07T03:28:19.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:18 smithi148 bash[82222]: cluster 2024-03-07T03:28:18.609244+0000 mgr.y (mgr.34101) 431 : cluster 0 pgmap v722: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 19 op/s 2024-03-07T03:28:19.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:18 smithi195 bash[66542]: cluster 2024-03-07T03:28:18.609244+0000 mgr.y (mgr.34101) 431 : cluster 0 pgmap v722: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 19 op/s 2024-03-07T03:28:21.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:20 smithi148 bash[79819]: cluster 2024-03-07T03:28:20.610642+0000 mgr.y (mgr.34101) 432 : cluster 0 pgmap v723: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 21 op/s 2024-03-07T03:28:21.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:20 smithi148 bash[82222]: cluster 2024-03-07T03:28:20.610642+0000 mgr.y (mgr.34101) 432 : cluster 0 pgmap v723: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 21 op/s 2024-03-07T03:28:21.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:20 smithi195 bash[66542]: cluster 2024-03-07T03:28:20.610642+0000 mgr.y (mgr.34101) 432 : cluster 0 pgmap v723: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 21 op/s 2024-03-07T03:28:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:28:22 smithi195 bash[59134]: ts=2024-03-07T03:28:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:23.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:22 smithi148 bash[79819]: cluster 2024-03-07T03:28:22.611461+0000 mgr.y (mgr.34101) 433 : cluster 0 pgmap v724: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:23.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:22 smithi148 bash[82222]: cluster 2024-03-07T03:28:22.611461+0000 mgr.y (mgr.34101) 433 : cluster 0 pgmap v724: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:23.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:22 smithi195 bash[66542]: cluster 2024-03-07T03:28:22.611461+0000 mgr.y (mgr.34101) 433 : cluster 0 pgmap v724: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:25.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:24 smithi148 bash[82222]: cluster 2024-03-07T03:28:24.612902+0000 mgr.y (mgr.34101) 434 : cluster 0 pgmap v725: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 24 op/s 2024-03-07T03:28:25.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:24 smithi148 bash[79819]: cluster 2024-03-07T03:28:24.612902+0000 mgr.y (mgr.34101) 434 : cluster 0 pgmap v725: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 24 op/s 2024-03-07T03:28:25.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:24 smithi195 bash[66542]: cluster 2024-03-07T03:28:24.612902+0000 mgr.y (mgr.34101) 434 : cluster 0 pgmap v725: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 188 KiB/s rd, 24 op/s 2024-03-07T03:28:27.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:26 smithi148 bash[82222]: cluster 2024-03-07T03:28:26.613989+0000 mgr.y (mgr.34101) 435 : cluster 0 pgmap v726: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:27.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:26 smithi148 bash[79819]: cluster 2024-03-07T03:28:26.613989+0000 mgr.y (mgr.34101) 435 : cluster 0 pgmap v726: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:26 smithi195 bash[66542]: cluster 2024-03-07T03:28:26.613989+0000 mgr.y (mgr.34101) 435 : cluster 0 pgmap v726: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:28:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:28:27] "GET /metrics HTTP/1.1" 200 60732 "" "Prometheus/2.43.0" 2024-03-07T03:28:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:27 smithi148 bash[82222]: audit 2024-03-07T03:28:27.319911+0000 mon.a (mon.0) 1116 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:27 smithi148 bash[79819]: audit 2024-03-07T03:28:27.319911+0000 mon.a (mon.0) 1116 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:27 smithi195 bash[66542]: audit 2024-03-07T03:28:27.319911+0000 mon.a (mon.0) 1116 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:29.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:28 smithi148 bash[79819]: audit 2024-03-07T03:28:28.122789+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:28:29.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:28 smithi148 bash[79819]: cluster 2024-03-07T03:28:28.615118+0000 mgr.y (mgr.34101) 436 : cluster 0 pgmap v727: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:28:29.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:28 smithi148 bash[82222]: audit 2024-03-07T03:28:28.122789+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:28:29.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:28 smithi148 bash[82222]: cluster 2024-03-07T03:28:28.615118+0000 mgr.y (mgr.34101) 436 : cluster 0 pgmap v727: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:28:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:28 smithi195 bash[66542]: audit 2024-03-07T03:28:28.122789+0000 mon.a (mon.0) 1117 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:28:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:28 smithi195 bash[66542]: cluster 2024-03-07T03:28:28.615118+0000 mgr.y (mgr.34101) 436 : cluster 0 pgmap v727: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:28:31.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:30 smithi148 bash[82222]: cluster 2024-03-07T03:28:30.616726+0000 mgr.y (mgr.34101) 437 : cluster 0 pgmap v728: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 108 KiB/s rd, 13 op/s 2024-03-07T03:28:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:30 smithi148 bash[79819]: cluster 2024-03-07T03:28:30.616726+0000 mgr.y (mgr.34101) 437 : cluster 0 pgmap v728: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 108 KiB/s rd, 13 op/s 2024-03-07T03:28:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:30 smithi195 bash[66542]: cluster 2024-03-07T03:28:30.616726+0000 mgr.y (mgr.34101) 437 : cluster 0 pgmap v728: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 108 KiB/s rd, 13 op/s 2024-03-07T03:28:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:28:32 smithi195 bash[59134]: ts=2024-03-07T03:28:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:33.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:32 smithi148 bash[79819]: cluster 2024-03-07T03:28:32.618041+0000 mgr.y (mgr.34101) 438 : cluster 0 pgmap v729: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 8 op/s 2024-03-07T03:28:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:32 smithi148 bash[82222]: cluster 2024-03-07T03:28:32.618041+0000 mgr.y (mgr.34101) 438 : cluster 0 pgmap v729: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 8 op/s 2024-03-07T03:28:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:32 smithi195 bash[66542]: cluster 2024-03-07T03:28:32.618041+0000 mgr.y (mgr.34101) 438 : cluster 0 pgmap v729: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 103 KiB/s rd, 8 op/s 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: audit 2024-03-07T03:28:33.911373+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: audit 2024-03-07T03:28:33.915942+0000 mon.a (mon.0) 1119 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: audit 2024-03-07T03:28:34.180500+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: audit 2024-03-07T03:28:34.190694+0000 mon.a (mon.0) 1121 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: cluster 2024-03-07T03:28:34.620841+0000 mgr.y (mgr.34101) 439 : cluster 0 pgmap v730: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: audit 2024-03-07T03:28:34.820478+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: audit 2024-03-07T03:28:34.822065+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[79819]: audit 2024-03-07T03:28:34.831422+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: audit 2024-03-07T03:28:33.911373+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: audit 2024-03-07T03:28:33.915942+0000 mon.a (mon.0) 1119 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: audit 2024-03-07T03:28:34.180500+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: audit 2024-03-07T03:28:34.190694+0000 mon.a (mon.0) 1121 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: cluster 2024-03-07T03:28:34.620841+0000 mgr.y (mgr.34101) 439 : cluster 0 pgmap v730: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:28:35.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: audit 2024-03-07T03:28:34.820478+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:28:35.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: audit 2024-03-07T03:28:34.822065+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:28:35.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:34 smithi148 bash[82222]: audit 2024-03-07T03:28:34.831422+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: audit 2024-03-07T03:28:33.911373+0000 mon.a (mon.0) 1118 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: audit 2024-03-07T03:28:33.915942+0000 mon.a (mon.0) 1119 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: audit 2024-03-07T03:28:34.180500+0000 mon.a (mon.0) 1120 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: audit 2024-03-07T03:28:34.190694+0000 mon.a (mon.0) 1121 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: cluster 2024-03-07T03:28:34.620841+0000 mgr.y (mgr.34101) 439 : cluster 0 pgmap v730: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: audit 2024-03-07T03:28:34.820478+0000 mon.a (mon.0) 1122 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: audit 2024-03-07T03:28:34.822065+0000 mon.a (mon.0) 1123 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:28:35.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:34 smithi195 bash[66542]: audit 2024-03-07T03:28:34.831422+0000 mon.a (mon.0) 1124 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:28:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:36 smithi148 bash[79819]: cluster 2024-03-07T03:28:36.622221+0000 mgr.y (mgr.34101) 440 : cluster 0 pgmap v731: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:37.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:36 smithi148 bash[82222]: cluster 2024-03-07T03:28:36.622221+0000 mgr.y (mgr.34101) 440 : cluster 0 pgmap v731: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:36 smithi195 bash[66542]: cluster 2024-03-07T03:28:36.622221+0000 mgr.y (mgr.34101) 440 : cluster 0 pgmap v731: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 8 op/s 2024-03-07T03:28:37.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:28:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:28:37] "GET /metrics HTTP/1.1" 200 60731 "" "Prometheus/2.43.0" 2024-03-07T03:28:39.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:38 smithi148 bash[79819]: cluster 2024-03-07T03:28:38.624138+0000 mgr.y (mgr.34101) 441 : cluster 0 pgmap v732: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 11 op/s 2024-03-07T03:28:39.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:38 smithi148 bash[82222]: cluster 2024-03-07T03:28:38.624138+0000 mgr.y (mgr.34101) 441 : cluster 0 pgmap v732: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 11 op/s 2024-03-07T03:28:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:38 smithi195 bash[66542]: cluster 2024-03-07T03:28:38.624138+0000 mgr.y (mgr.34101) 441 : cluster 0 pgmap v732: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 11 op/s 2024-03-07T03:28:41.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:40 smithi148 bash[79819]: cluster 2024-03-07T03:28:40.626385+0000 mgr.y (mgr.34101) 442 : cluster 0 pgmap v733: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:41.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:40 smithi148 bash[82222]: cluster 2024-03-07T03:28:40.626385+0000 mgr.y (mgr.34101) 442 : cluster 0 pgmap v733: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:41.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:40 smithi195 bash[66542]: cluster 2024-03-07T03:28:40.626385+0000 mgr.y (mgr.34101) 442 : cluster 0 pgmap v733: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-03-07T03:28:42.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:42 smithi195 bash[66542]: audit 2024-03-07T03:28:42.320438+0000 mon.a (mon.0) 1125 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:28:42 smithi195 bash[59134]: ts=2024-03-07T03:28:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:42.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:42 smithi148 bash[79819]: audit 2024-03-07T03:28:42.320438+0000 mon.a (mon.0) 1125 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:42 smithi148 bash[82222]: audit 2024-03-07T03:28:42.320438+0000 mon.a (mon.0) 1125 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:43.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:43 smithi148 bash[79819]: cluster 2024-03-07T03:28:42.627418+0000 mgr.y (mgr.34101) 443 : cluster 0 pgmap v734: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:28:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:43 smithi148 bash[82222]: cluster 2024-03-07T03:28:42.627418+0000 mgr.y (mgr.34101) 443 : cluster 0 pgmap v734: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:28:43.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:43 smithi195 bash[66542]: cluster 2024-03-07T03:28:42.627418+0000 mgr.y (mgr.34101) 443 : cluster 0 pgmap v734: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 15 op/s 2024-03-07T03:28:45.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:44 smithi148 bash[79819]: cluster 2024-03-07T03:28:44.630137+0000 mgr.y (mgr.34101) 444 : cluster 0 pgmap v735: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 85 B/s wr, 24 op/s 2024-03-07T03:28:45.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:44 smithi148 bash[82222]: cluster 2024-03-07T03:28:44.630137+0000 mgr.y (mgr.34101) 444 : cluster 0 pgmap v735: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 85 B/s wr, 24 op/s 2024-03-07T03:28:45.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:44 smithi195 bash[66542]: cluster 2024-03-07T03:28:44.630137+0000 mgr.y (mgr.34101) 444 : cluster 0 pgmap v735: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 85 B/s wr, 24 op/s 2024-03-07T03:28:47.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:46 smithi148 bash[79819]: cluster 2024-03-07T03:28:46.632099+0000 mgr.y (mgr.34101) 445 : cluster 0 pgmap v736: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:28:47.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:46 smithi148 bash[82222]: cluster 2024-03-07T03:28:46.632099+0000 mgr.y (mgr.34101) 445 : cluster 0 pgmap v736: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:28:47.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:46 smithi195 bash[66542]: cluster 2024-03-07T03:28:46.632099+0000 mgr.y (mgr.34101) 445 : cluster 0 pgmap v736: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-03-07T03:28:47.111 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.EncryptionLoadFormatMismatch (130594 ms) 2024-03-07T03:28:47.111 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EncryptedResize 2024-03-07T03:28:47.111 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:28:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:28:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:28:47] "GET /metrics HTTP/1.1" 200 60731 "" "Prometheus/2.43.0" 2024-03-07T03:28:49.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:48 smithi148 bash[79819]: cluster 2024-03-07T03:28:48.633648+0000 mgr.y (mgr.34101) 446 : cluster 0 pgmap v737: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 341 B/s wr, 17 op/s 2024-03-07T03:28:49.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:48 smithi148 bash[82222]: cluster 2024-03-07T03:28:48.633648+0000 mgr.y (mgr.34101) 446 : cluster 0 pgmap v737: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 341 B/s wr, 17 op/s 2024-03-07T03:28:49.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:48 smithi195 bash[66542]: cluster 2024-03-07T03:28:48.633648+0000 mgr.y (mgr.34101) 446 : cluster 0 pgmap v737: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 341 B/s wr, 17 op/s 2024-03-07T03:28:51.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:50 smithi148 bash[79819]: cluster 2024-03-07T03:28:50.635181+0000 mgr.y (mgr.34101) 447 : cluster 0 pgmap v738: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 108 KiB/s rd, 341 B/s wr, 14 op/s 2024-03-07T03:28:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:50 smithi148 bash[82222]: cluster 2024-03-07T03:28:50.635181+0000 mgr.y (mgr.34101) 447 : cluster 0 pgmap v738: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 108 KiB/s rd, 341 B/s wr, 14 op/s 2024-03-07T03:28:51.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:50 smithi195 bash[66542]: cluster 2024-03-07T03:28:50.635181+0000 mgr.y (mgr.34101) 447 : cluster 0 pgmap v738: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 108 KiB/s rd, 341 B/s wr, 14 op/s 2024-03-07T03:28:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:28:52 smithi195 bash[59134]: ts=2024-03-07T03:28:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:28:53.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:52 smithi148 bash[79819]: cluster 2024-03-07T03:28:52.636576+0000 mgr.y (mgr.34101) 448 : cluster 0 pgmap v739: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 426 B/s wr, 11 op/s 2024-03-07T03:28:53.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:52 smithi148 bash[82222]: cluster 2024-03-07T03:28:52.636576+0000 mgr.y (mgr.34101) 448 : cluster 0 pgmap v739: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 426 B/s wr, 11 op/s 2024-03-07T03:28:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:52 smithi195 bash[66542]: cluster 2024-03-07T03:28:52.636576+0000 mgr.y (mgr.34101) 448 : cluster 0 pgmap v739: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 426 B/s wr, 11 op/s 2024-03-07T03:28:55.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:54 smithi148 bash[79819]: cluster 2024-03-07T03:28:54.638427+0000 mgr.y (mgr.34101) 449 : cluster 0 pgmap v740: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 426 B/s wr, 11 op/s 2024-03-07T03:28:55.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:54 smithi148 bash[82222]: cluster 2024-03-07T03:28:54.638427+0000 mgr.y (mgr.34101) 449 : cluster 0 pgmap v740: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 426 B/s wr, 11 op/s 2024-03-07T03:28:55.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:54 smithi195 bash[66542]: cluster 2024-03-07T03:28:54.638427+0000 mgr.y (mgr.34101) 449 : cluster 0 pgmap v740: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 426 B/s wr, 11 op/s 2024-03-07T03:28:57.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:56 smithi148 bash[79819]: cluster 2024-03-07T03:28:56.641452+0000 mgr.y (mgr.34101) 450 : cluster 0 pgmap v741: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 341 B/s wr, 3 op/s 2024-03-07T03:28:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:56 smithi148 bash[82222]: cluster 2024-03-07T03:28:56.641452+0000 mgr.y (mgr.34101) 450 : cluster 0 pgmap v741: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 341 B/s wr, 3 op/s 2024-03-07T03:28:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:56 smithi195 bash[66542]: cluster 2024-03-07T03:28:56.641452+0000 mgr.y (mgr.34101) 450 : cluster 0 pgmap v741: 425 pgs: 425 active+clean; 149 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 341 B/s wr, 3 op/s 2024-03-07T03:28:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:28:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:28:57] "GET /metrics HTTP/1.1" 200 60731 "" "Prometheus/2.43.0" 2024-03-07T03:28:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:57 smithi148 bash[79819]: audit 2024-03-07T03:28:57.320782+0000 mon.a (mon.0) 1126 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:57 smithi148 bash[82222]: audit 2024-03-07T03:28:57.320782+0000 mon.a (mon.0) 1126 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:57 smithi195 bash[66542]: audit 2024-03-07T03:28:57.320782+0000 mon.a (mon.0) 1126 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:28:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:28:58 smithi148 bash[79819]: cluster 2024-03-07T03:28:58.642714+0000 mgr.y (mgr.34101) 451 : cluster 0 pgmap v742: 425 pgs: 425 active+clean; 153 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 342 KiB/s wr, 7 op/s 2024-03-07T03:28:59.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:28:58 smithi148 bash[82222]: cluster 2024-03-07T03:28:58.642714+0000 mgr.y (mgr.34101) 451 : cluster 0 pgmap v742: 425 pgs: 425 active+clean; 153 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 342 KiB/s wr, 7 op/s 2024-03-07T03:28:59.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:28:58 smithi195 bash[66542]: cluster 2024-03-07T03:28:58.642714+0000 mgr.y (mgr.34101) 451 : cluster 0 pgmap v742: 425 pgs: 425 active+clean; 153 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 342 KiB/s wr, 7 op/s 2024-03-07T03:29:01.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:00 smithi148 bash[79819]: cluster 2024-03-07T03:29:00.644284+0000 mgr.y (mgr.34101) 452 : cluster 0 pgmap v743: 425 pgs: 425 active+clean; 169 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-03-07T03:29:01.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:00 smithi148 bash[82222]: cluster 2024-03-07T03:29:00.644284+0000 mgr.y (mgr.34101) 452 : cluster 0 pgmap v743: 425 pgs: 425 active+clean; 169 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-03-07T03:29:01.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:00 smithi195 bash[66542]: cluster 2024-03-07T03:29:00.644284+0000 mgr.y (mgr.34101) 452 : cluster 0 pgmap v743: 425 pgs: 425 active+clean; 169 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 1.6 MiB/s wr, 7 op/s 2024-03-07T03:29:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:29:02 smithi195 bash[59134]: ts=2024-03-07T03:29:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:03.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:02 smithi148 bash[79819]: cluster 2024-03-07T03:29:02.645263+0000 mgr.y (mgr.34101) 453 : cluster 0 pgmap v744: 425 pgs: 425 active+clean; 169 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-03-07T03:29:03.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:02 smithi148 bash[82222]: cluster 2024-03-07T03:29:02.645263+0000 mgr.y (mgr.34101) 453 : cluster 0 pgmap v744: 425 pgs: 425 active+clean; 169 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-03-07T03:29:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:02 smithi195 bash[66542]: cluster 2024-03-07T03:29:02.645263+0000 mgr.y (mgr.34101) 453 : cluster 0 pgmap v744: 425 pgs: 425 active+clean; 169 MiB data, 864 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-03-07T03:29:03.065 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.EncryptedResize (15954 ms) 2024-03-07T03:29:03.066 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EncryptedFlattenSmallData 2024-03-07T03:29:03.066 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:05.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:04 smithi148 bash[79819]: cluster 2024-03-07T03:29:04.647315+0000 mgr.y (mgr.34101) 454 : cluster 0 pgmap v745: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 1.6 MiB/s wr, 25 op/s 2024-03-07T03:29:05.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:04 smithi148 bash[82222]: cluster 2024-03-07T03:29:04.647315+0000 mgr.y (mgr.34101) 454 : cluster 0 pgmap v745: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 1.6 MiB/s wr, 25 op/s 2024-03-07T03:29:05.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:04 smithi195 bash[66542]: cluster 2024-03-07T03:29:04.647315+0000 mgr.y (mgr.34101) 454 : cluster 0 pgmap v745: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 1.6 MiB/s wr, 25 op/s 2024-03-07T03:29:07.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:06 smithi148 bash[79819]: cluster 2024-03-07T03:29:06.648686+0000 mgr.y (mgr.34101) 455 : cluster 0 pgmap v746: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:29:07.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:06 smithi148 bash[82222]: cluster 2024-03-07T03:29:06.648686+0000 mgr.y (mgr.34101) 455 : cluster 0 pgmap v746: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:29:07.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:06 smithi195 bash[66542]: cluster 2024-03-07T03:29:06.648686+0000 mgr.y (mgr.34101) 455 : cluster 0 pgmap v746: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:29:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:29:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:29:07] "GET /metrics HTTP/1.1" 200 60728 "" "Prometheus/2.43.0" 2024-03-07T03:29:09.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:08 smithi148 bash[79819]: cluster 2024-03-07T03:29:08.649999+0000 mgr.y (mgr.34101) 456 : cluster 0 pgmap v747: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:29:09.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:08 smithi148 bash[82222]: cluster 2024-03-07T03:29:08.649999+0000 mgr.y (mgr.34101) 456 : cluster 0 pgmap v747: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:29:09.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:08 smithi195 bash[66542]: cluster 2024-03-07T03:29:08.649999+0000 mgr.y (mgr.34101) 456 : cluster 0 pgmap v747: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 93 KiB/s rd, 1.6 MiB/s wr, 29 op/s 2024-03-07T03:29:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:11 smithi148 bash[79819]: cluster 2024-03-07T03:29:10.464884+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2024-03-07T03:29:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:11 smithi148 bash[79819]: cluster 2024-03-07T03:29:10.650712+0000 mgr.y (mgr.34101) 457 : cluster 0 pgmap v749: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.7 KiB/s wr, 29 op/s 2024-03-07T03:29:11.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:11 smithi148 bash[82222]: cluster 2024-03-07T03:29:10.464884+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2024-03-07T03:29:11.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:11 smithi148 bash[82222]: cluster 2024-03-07T03:29:10.650712+0000 mgr.y (mgr.34101) 457 : cluster 0 pgmap v749: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.7 KiB/s wr, 29 op/s 2024-03-07T03:29:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:11 smithi195 bash[66542]: cluster 2024-03-07T03:29:10.464884+0000 mon.a (mon.0) 1127 : cluster 0 osdmap e673: 8 total, 8 up, 8 in 2024-03-07T03:29:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:11 smithi195 bash[66542]: cluster 2024-03-07T03:29:10.650712+0000 mgr.y (mgr.34101) 457 : cluster 0 pgmap v749: 425 pgs: 425 active+clean; 165 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 1.7 KiB/s wr, 29 op/s 2024-03-07T03:29:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:12 smithi195 bash[66542]: audit 2024-03-07T03:29:12.322213+0000 mon.a (mon.0) 1128 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:29:12 smithi195 bash[59134]: ts=2024-03-07T03:29:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:12 smithi148 bash[79819]: audit 2024-03-07T03:29:12.322213+0000 mon.a (mon.0) 1128 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:12 smithi148 bash[82222]: audit 2024-03-07T03:29:12.322213+0000 mon.a (mon.0) 1128 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:13 smithi148 bash[79819]: cluster 2024-03-07T03:29:12.651704+0000 mgr.y (mgr.34101) 458 : cluster 0 pgmap v750: 425 pgs: 425 active+clean; 169 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 412 KiB/s wr, 22 op/s 2024-03-07T03:29:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:13 smithi148 bash[82222]: cluster 2024-03-07T03:29:12.651704+0000 mgr.y (mgr.34101) 458 : cluster 0 pgmap v750: 425 pgs: 425 active+clean; 169 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 412 KiB/s wr, 22 op/s 2024-03-07T03:29:13.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:13 smithi195 bash[66542]: cluster 2024-03-07T03:29:12.651704+0000 mgr.y (mgr.34101) 458 : cluster 0 pgmap v750: 425 pgs: 425 active+clean; 169 MiB data, 872 MiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 412 KiB/s wr, 22 op/s 2024-03-07T03:29:15.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:14 smithi148 bash[79819]: cluster 2024-03-07T03:29:14.653424+0000 mgr.y (mgr.34101) 459 : cluster 0 pgmap v751: 425 pgs: 425 active+clean; 177 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 1.2 MiB/s wr, 33 op/s 2024-03-07T03:29:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:14 smithi148 bash[82222]: cluster 2024-03-07T03:29:14.653424+0000 mgr.y (mgr.34101) 459 : cluster 0 pgmap v751: 425 pgs: 425 active+clean; 177 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 1.2 MiB/s wr, 33 op/s 2024-03-07T03:29:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:14 smithi195 bash[66542]: cluster 2024-03-07T03:29:14.653424+0000 mgr.y (mgr.34101) 459 : cluster 0 pgmap v751: 425 pgs: 425 active+clean; 177 MiB data, 904 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 1.2 MiB/s wr, 33 op/s 2024-03-07T03:29:16.193 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.EncryptedFlattenSmallData (13128 ms) 2024-03-07T03:29:16.193 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-03-07T03:29:16.193 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:17.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:16 smithi148 bash[79819]: cluster 2024-03-07T03:29:16.654996+0000 mgr.y (mgr.34101) 460 : cluster 0 pgmap v752: 425 pgs: 425 active+clean; 185 MiB data, 924 MiB used, 714 GiB / 715 GiB avail; 905 KiB/s rd, 2.0 MiB/s wr, 37 op/s 2024-03-07T03:29:17.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:16 smithi148 bash[82222]: cluster 2024-03-07T03:29:16.654996+0000 mgr.y (mgr.34101) 460 : cluster 0 pgmap v752: 425 pgs: 425 active+clean; 185 MiB data, 924 MiB used, 714 GiB / 715 GiB avail; 905 KiB/s rd, 2.0 MiB/s wr, 37 op/s 2024-03-07T03:29:17.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:16 smithi195 bash[66542]: cluster 2024-03-07T03:29:16.654996+0000 mgr.y (mgr.34101) 460 : cluster 0 pgmap v752: 425 pgs: 425 active+clean; 185 MiB data, 924 MiB used, 714 GiB / 715 GiB avail; 905 KiB/s rd, 2.0 MiB/s wr, 37 op/s 2024-03-07T03:29:17.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:29:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:29:17] "GET /metrics HTTP/1.1" 200 60730 "" "Prometheus/2.43.0" 2024-03-07T03:29:19.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:18 smithi148 bash[79819]: cluster 2024-03-07T03:29:18.657250+0000 mgr.y (mgr.34101) 461 : cluster 0 pgmap v753: 425 pgs: 425 active+clean; 193 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.8 MiB/s wr, 43 op/s 2024-03-07T03:29:19.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:18 smithi148 bash[82222]: cluster 2024-03-07T03:29:18.657250+0000 mgr.y (mgr.34101) 461 : cluster 0 pgmap v753: 425 pgs: 425 active+clean; 193 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.8 MiB/s wr, 43 op/s 2024-03-07T03:29:19.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:18 smithi195 bash[66542]: cluster 2024-03-07T03:29:18.657250+0000 mgr.y (mgr.34101) 461 : cluster 0 pgmap v753: 425 pgs: 425 active+clean; 193 MiB data, 932 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 2.8 MiB/s wr, 43 op/s 2024-03-07T03:29:21.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:20 smithi148 bash[79819]: cluster 2024-03-07T03:29:20.659681+0000 mgr.y (mgr.34101) 462 : cluster 0 pgmap v754: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 47 op/s 2024-03-07T03:29:21.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:20 smithi148 bash[82222]: cluster 2024-03-07T03:29:20.659681+0000 mgr.y (mgr.34101) 462 : cluster 0 pgmap v754: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 47 op/s 2024-03-07T03:29:21.053 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:21.053 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:21.053 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:21.053 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:21.053 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:21.053 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.053 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.054 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.055 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:21.056 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.057 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:21.058 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.059 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:21.060 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 512~512 = 512 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 1536~512 = 0 2024-03-07T03:29:21.061 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.062 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.063 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:29:21.064 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.065 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:29:21.066 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.067 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.068 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.069 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.070 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:21.071 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:21.072 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:21.072 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:21.072 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:21.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:20 smithi195 bash[66542]: cluster 2024-03-07T03:29:20.659681+0000 mgr.y (mgr.34101) 462 : cluster 0 pgmap v754: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 47 op/s 2024-03-07T03:29:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:29:22 smithi195 bash[59134]: ts=2024-03-07T03:29:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:23.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:22 smithi148 bash[79819]: cluster 2024-03-07T03:29:22.661221+0000 mgr.y (mgr.34101) 463 : cluster 0 pgmap v755: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.8 MiB/s wr, 426 op/s 2024-03-07T03:29:23.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:22 smithi148 bash[82222]: cluster 2024-03-07T03:29:22.661221+0000 mgr.y (mgr.34101) 463 : cluster 0 pgmap v755: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.8 MiB/s wr, 426 op/s 2024-03-07T03:29:23.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:22 smithi195 bash[66542]: cluster 2024-03-07T03:29:22.661221+0000 mgr.y (mgr.34101) 463 : cluster 0 pgmap v755: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.8 MiB/s wr, 426 op/s 2024-03-07T03:29:25.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:24 smithi148 bash[79819]: cluster 2024-03-07T03:29:24.663308+0000 mgr.y (mgr.34101) 464 : cluster 0 pgmap v756: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.4 MiB/s wr, 421 op/s 2024-03-07T03:29:25.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:24 smithi148 bash[82222]: cluster 2024-03-07T03:29:24.663308+0000 mgr.y (mgr.34101) 464 : cluster 0 pgmap v756: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.4 MiB/s wr, 421 op/s 2024-03-07T03:29:25.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:24 smithi195 bash[66542]: cluster 2024-03-07T03:29:24.663308+0000 mgr.y (mgr.34101) 464 : cluster 0 pgmap v756: 425 pgs: 425 active+clean; 197 MiB data, 966 MiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.4 MiB/s wr, 421 op/s 2024-03-07T03:29:25.235 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.236 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.237 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.238 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.239 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.241 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.242 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.243 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:25.244 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:25.245 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (9042 ms) 2024-03-07T03:29:25.245 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-03-07T03:29:26.821 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:26 smithi195 bash[66542]: cluster 2024-03-07T03:29:25.813166+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2024-03-07T03:29:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:26 smithi195 bash[66542]: audit 2024-03-07T03:29:25.815080+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.148:0/690235984' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:26 smithi195 bash[66542]: audit 2024-03-07T03:29:25.821809+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:26 smithi195 bash[66542]: cluster 2024-03-07T03:29:26.664904+0000 mgr.y (mgr.34101) 465 : cluster 0 pgmap v758: 457 pgs: 4 creating+peering, 28 unknown, 425 active+clean; 199 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 849 op/s 2024-03-07T03:29:27.169 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[79819]: cluster 2024-03-07T03:29:25.813166+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2024-03-07T03:29:27.169 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[79819]: audit 2024-03-07T03:29:25.815080+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.148:0/690235984' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:27.169 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[79819]: audit 2024-03-07T03:29:25.821809+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:27.169 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[79819]: cluster 2024-03-07T03:29:26.664904+0000 mgr.y (mgr.34101) 465 : cluster 0 pgmap v758: 457 pgs: 4 creating+peering, 28 unknown, 425 active+clean; 199 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 849 op/s 2024-03-07T03:29:27.169 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[82222]: cluster 2024-03-07T03:29:25.813166+0000 mon.a (mon.0) 1129 : cluster 0 osdmap e674: 8 total, 8 up, 8 in 2024-03-07T03:29:27.169 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[82222]: audit 2024-03-07T03:29:25.815080+0000 mon.c (mon.2) 60 : audit 1 from='client.? 172.21.15.148:0/690235984' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:27.169 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[82222]: audit 2024-03-07T03:29:25.821809+0000 mon.a (mon.0) 1130 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:27.170 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:26 smithi148 bash[82222]: cluster 2024-03-07T03:29:26.664904+0000 mgr.y (mgr.34101) 465 : cluster 0 pgmap v758: 457 pgs: 4 creating+peering, 28 unknown, 425 active+clean; 199 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.4 MiB/s wr, 849 op/s 2024-03-07T03:29:27.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:29:27] "GET /metrics HTTP/1.1" 200 60730 "" "Prometheus/2.43.0" 2024-03-07T03:29:28.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[79819]: cluster 2024-03-07T03:29:26.803838+0000 mon.a (mon.0) 1131 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:28.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[79819]: audit 2024-03-07T03:29:26.816112+0000 mon.a (mon.0) 1132 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:28.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[79819]: cluster 2024-03-07T03:29:26.824762+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2024-03-07T03:29:28.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[79819]: audit 2024-03-07T03:29:27.328490+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:28.292 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[79819]: audit 2024-03-07T03:29:27.331855+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:28.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[82222]: cluster 2024-03-07T03:29:26.803838+0000 mon.a (mon.0) 1131 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:28.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[82222]: audit 2024-03-07T03:29:26.816112+0000 mon.a (mon.0) 1132 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:28.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[82222]: cluster 2024-03-07T03:29:26.824762+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2024-03-07T03:29:28.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[82222]: audit 2024-03-07T03:29:27.328490+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:28.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:27 smithi148 bash[82222]: audit 2024-03-07T03:29:27.331855+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:27 smithi195 bash[66542]: cluster 2024-03-07T03:29:26.803838+0000 mon.a (mon.0) 1131 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:29:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:27 smithi195 bash[66542]: audit 2024-03-07T03:29:26.816112+0000 mon.a (mon.0) 1132 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:27 smithi195 bash[66542]: cluster 2024-03-07T03:29:26.824762+0000 mon.a (mon.0) 1133 : cluster 0 osdmap e675: 8 total, 8 up, 8 in 2024-03-07T03:29:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:27 smithi195 bash[66542]: audit 2024-03-07T03:29:27.328490+0000 mon.a (mon.0) 1134 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:27 smithi195 bash[66542]: audit 2024-03-07T03:29:27.331855+0000 mon.a (mon.0) 1135 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:28.990 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:28.990 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:28.990 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:28.991 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:28.992 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.993 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:28.994 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 512~512 = 512 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 1536~512 = 0 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:28.995 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestDataPoolIO (3755 ms) 2024-03-07T03:29:28.996 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-03-07T03:29:28.996 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.053 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (63 ms) 2024-03-07T03:29:29.053 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-03-07T03:29:29.053 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.113 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (60 ms) 2024-03-07T03:29:29.113 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-03-07T03:29:29.114 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.186 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (73 ms) 2024-03-07T03:29:29.186 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-03-07T03:29:29.186 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (72 ms) 2024-03-07T03:29:29.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-03-07T03:29:29.258 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:28 smithi148 bash[79819]: cluster 2024-03-07T03:29:27.827350+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2024-03-07T03:29:29.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:28 smithi148 bash[79819]: cluster 2024-03-07T03:29:28.667033+0000 mgr.y (mgr.34101) 466 : cluster 0 pgmap v761: 457 pgs: 4 creating+peering, 18 unknown, 435 active+clean; 199 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 304 KiB/s rd, 152 KiB/s wr, 624 op/s 2024-03-07T03:29:29.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:28 smithi148 bash[82222]: cluster 2024-03-07T03:29:27.827350+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2024-03-07T03:29:29.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:28 smithi148 bash[82222]: cluster 2024-03-07T03:29:28.667033+0000 mgr.y (mgr.34101) 466 : cluster 0 pgmap v761: 457 pgs: 4 creating+peering, 18 unknown, 435 active+clean; 199 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 304 KiB/s rd, 152 KiB/s wr, 624 op/s 2024-03-07T03:29:29.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:28 smithi195 bash[66542]: cluster 2024-03-07T03:29:27.827350+0000 mon.a (mon.0) 1136 : cluster 0 osdmap e676: 8 total, 8 up, 8 in 2024-03-07T03:29:29.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:28 smithi195 bash[66542]: cluster 2024-03-07T03:29:28.667033+0000 mgr.y (mgr.34101) 466 : cluster 0 pgmap v761: 457 pgs: 4 creating+peering, 18 unknown, 435 active+clean; 199 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 304 KiB/s rd, 152 KiB/s wr, 624 op/s 2024-03-07T03:29:29.394 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (136 ms) 2024-03-07T03:29:29.394 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-03-07T03:29:29.394 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.518 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (124 ms) 2024-03-07T03:29:29.519 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-03-07T03:29:29.519 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.651 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (133 ms) 2024-03-07T03:29:29.651 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-03-07T03:29:29.651 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:29.788 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (137 ms) 2024-03-07T03:29:29.788 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-03-07T03:29:29.788 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:30.212 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (424 ms) 2024-03-07T03:29:30.213 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-03-07T03:29:30.213 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:30.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:29 smithi148 bash[79819]: cluster 2024-03-07T03:29:28.841965+0000 mon.a (mon.0) 1137 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:30.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:29 smithi148 bash[79819]: cluster 2024-03-07T03:29:28.858234+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2024-03-07T03:29:30.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:29 smithi148 bash[82222]: cluster 2024-03-07T03:29:28.841965+0000 mon.a (mon.0) 1137 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:30.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:29 smithi148 bash[82222]: cluster 2024-03-07T03:29:28.858234+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2024-03-07T03:29:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:29 smithi195 bash[66542]: cluster 2024-03-07T03:29:28.841965+0000 mon.a (mon.0) 1137 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:29:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:29 smithi195 bash[66542]: cluster 2024-03-07T03:29:28.858234+0000 mon.a (mon.0) 1138 : cluster 0 osdmap e677: 8 total, 8 up, 8 in 2024-03-07T03:29:30.602 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (390 ms) 2024-03-07T03:29:30.602 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-03-07T03:29:30.602 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:30.667 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (65 ms) 2024-03-07T03:29:30.667 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-03-07T03:29:30.667 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:30.740 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (72 ms) 2024-03-07T03:29:30.740 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-03-07T03:29:30.740 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:30.812 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (73 ms) 2024-03-07T03:29:30.812 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-03-07T03:29:30.812 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:30.885 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (73 ms) 2024-03-07T03:29:30.885 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-03-07T03:29:30.885 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:30.945 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:30.945 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 0~1048576 = 0 2024-03-07T03:29:30.946 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:30.946 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 0~4194304 = 0 2024-03-07T03:29:30.946 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:30.946 INFO:tasks.workunit.client.0.smithi148.stdout:aio discard: 3145728~1048576 = 0 2024-03-07T03:29:30.946 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (60 ms) 2024-03-07T03:29:30.946 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestFUA 2024-03-07T03:29:30.946 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:31.237 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.237 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.238 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.239 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.240 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestFUA (292 ms) 2024-03-07T03:29:31.241 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-03-07T03:29:31.242 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.282 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:31.283 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:30 smithi148 bash[79819]: cluster 2024-03-07T03:29:30.669431+0000 mgr.y (mgr.34101) 467 : cluster 0 pgmap v763: 457 pgs: 4 creating+peering, 453 active+clean; 203 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.0 MiB/s wr, 789 op/s 2024-03-07T03:29:31.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:30 smithi148 bash[82222]: cluster 2024-03-07T03:29:30.669431+0000 mgr.y (mgr.34101) 467 : cluster 0 pgmap v763: 457 pgs: 4 creating+peering, 453 active+clean; 203 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.0 MiB/s wr, 789 op/s 2024-03-07T03:29:31.311 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.311 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:31.311 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:31.311 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:31.311 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:31.311 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:start compare and write 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:compare and wrote: 512 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:31.312 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:30 smithi195 bash[66542]: cluster 2024-03-07T03:29:30.669431+0000 mgr.y (mgr.34101) 467 : cluster 0 pgmap v763: 457 pgs: 4 creating+peering, 453 active+clean; 203 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.0 MiB/s wr, 789 op/s 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:31.316 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.318 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.318 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:31.318 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.318 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:31.318 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.320 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.320 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:31.320 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.320 INFO:tasks.workunit.client.0.smithi148.stdout:started aio compare and write 2024-03-07T03:29:31.320 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:finished aio compare and write 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.322 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.323 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:discard: 512~512 2024-03-07T03:29:31.324 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:29:32 smithi195 bash[59134]: ts=2024-03-07T03:29:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:32 smithi148 bash[82222]: cluster 2024-03-07T03:29:32.670485+0000 mgr.y (mgr.34101) 468 : cluster 0 pgmap v764: 457 pgs: 457 active+clean; 215 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 66 op/s 2024-03-07T03:29:33.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:32 smithi148 bash[79819]: cluster 2024-03-07T03:29:32.670485+0000 mgr.y (mgr.34101) 468 : cluster 0 pgmap v764: 457 pgs: 457 active+clean; 215 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 66 op/s 2024-03-07T03:29:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:32 smithi195 bash[66542]: cluster 2024-03-07T03:29:32.670485+0000 mgr.y (mgr.34101) 468 : cluster 0 pgmap v764: 457 pgs: 457 active+clean; 215 MiB data, 993 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 2.7 MiB/s wr, 66 op/s 2024-03-07T03:29:33.716 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:33.716 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:29:33.717 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.718 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.719 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.720 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:29:33.721 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.722 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.723 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.724 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.724 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.724 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:33.740 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:33.740 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:33.740 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.740 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.740 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.740 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.740 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:33.765 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.766 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:33.833 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:33.833 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:33.833 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.833 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.834 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.834 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.834 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:33.985 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:33.985 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:33.985 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:33.985 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:33.986 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:33.986 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:33.986 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:34.078 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.079 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:34.220 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:34.221 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:34.221 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:34.221 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:34.221 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.221 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.221 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:34.360 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:34.360 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:34.360 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:34.360 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:34.360 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.360 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.361 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:34.519 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:34.519 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:34.519 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:34.519 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:34.519 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.519 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.519 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:34.677 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:34.677 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:34.677 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.678 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:34.895 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:34.895 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:34.895 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:34.895 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:34.895 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:34.895 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:34.896 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:35.033 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:35.034 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:35.034 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:35.034 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:35.034 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.034 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.034 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:35.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:34 smithi148 bash[79819]: cluster 2024-03-07T03:29:34.674185+0000 mgr.y (mgr.34101) 469 : cluster 0 pgmap v765: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 789 op/s 2024-03-07T03:29:35.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:34 smithi148 bash[82222]: cluster 2024-03-07T03:29:34.674185+0000 mgr.y (mgr.34101) 469 : cluster 0 pgmap v765: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 789 op/s 2024-03-07T03:29:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:34 smithi195 bash[66542]: cluster 2024-03-07T03:29:34.674185+0000 mgr.y (mgr.34101) 469 : cluster 0 pgmap v765: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 7.3 MiB/s wr, 789 op/s 2024-03-07T03:29:35.152 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:35.153 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:35.153 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:35.153 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:35.153 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.153 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.153 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:35.277 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.278 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:35.416 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:35.417 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:35.417 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:35.417 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:35.417 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.417 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.417 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:35.545 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:35.545 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:35.545 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:35.545 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:35.545 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.545 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.545 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:35.691 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:35.691 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:35.691 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:35.691 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:35.691 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.691 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.691 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:35.913 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:35.913 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestIOPP (4676 ms) 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-03-07T03:29:35.914 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:35.955 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (42 ms) 2024-03-07T03:29:35.955 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-03-07T03:29:35.956 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.005 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (50 ms) 2024-03-07T03:29:36.006 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-03-07T03:29:36.006 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:35 smithi148 bash[79819]: audit 2024-03-07T03:29:34.910886+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:35 smithi148 bash[82222]: audit 2024-03-07T03:29:34.910886+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:36.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:35 smithi195 bash[66542]: audit 2024-03-07T03:29:34.910886+0000 mon.a (mon.0) 1139 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:29:36.067 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (62 ms) 2024-03-07T03:29:36.067 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-03-07T03:29:36.067 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.127 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (60 ms) 2024-03-07T03:29:36.127 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-03-07T03:29:36.127 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.191 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (64 ms) 2024-03-07T03:29:36.191 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-03-07T03:29:36.192 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.243 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:36.252 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (61 ms) 2024-03-07T03:29:36.252 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-03-07T03:29:36.252 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.311 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (59 ms) 2024-03-07T03:29:36.311 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-03-07T03:29:36.312 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.363 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:36.373 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (62 ms) 2024-03-07T03:29:36.373 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-03-07T03:29:36.373 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.435 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (62 ms) 2024-03-07T03:29:36.435 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-03-07T03:29:36.435 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.487 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:36.497 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (62 ms) 2024-03-07T03:29:36.497 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-03-07T03:29:36.497 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.560 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (63 ms) 2024-03-07T03:29:36.560 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-03-07T03:29:36.560 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.615 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:36.625 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (65 ms) 2024-03-07T03:29:36.625 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-03-07T03:29:36.625 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.763 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (137 ms) 2024-03-07T03:29:36.763 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-03-07T03:29:36.763 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:36.851 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:36.878 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (115 ms) 2024-03-07T03:29:36.878 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-03-07T03:29:36.878 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:37.019 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (141 ms) 2024-03-07T03:29:37.020 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-03-07T03:29:37.020 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:37.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:36 smithi148 bash[79819]: cluster 2024-03-07T03:29:36.676546+0000 mgr.y (mgr.34101) 470 : cluster 0 pgmap v766: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.5 MiB/s wr, 712 op/s 2024-03-07T03:29:37.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:36 smithi148 bash[82222]: cluster 2024-03-07T03:29:36.676546+0000 mgr.y (mgr.34101) 470 : cluster 0 pgmap v766: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.5 MiB/s wr, 712 op/s 2024-03-07T03:29:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:36 smithi195 bash[66542]: cluster 2024-03-07T03:29:36.676546+0000 mgr.y (mgr.34101) 470 : cluster 0 pgmap v766: 457 pgs: 457 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 6.5 MiB/s wr, 712 op/s 2024-03-07T03:29:37.145 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:37.207 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (188 ms) 2024-03-07T03:29:37.208 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-03-07T03:29:37.208 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:37.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:29:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:29:37] "GET /metrics HTTP/1.1" 200 63100 "" "Prometheus/2.43.0" 2024-03-07T03:29:37.672 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (465 ms) 2024-03-07T03:29:37.672 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-03-07T03:29:37.672 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:38.025 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:38.057 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (385 ms) 2024-03-07T03:29:38.057 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-03-07T03:29:38.057 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:38.121 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:38.121 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:38.121 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:38.121 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:38.121 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:29:38.121 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.122 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:38.122 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:38.123 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:38.123 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:38.123 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.124 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:38.124 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:38.125 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:38.125 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:38.125 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.126 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:38.126 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:38.128 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:38.128 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:38.128 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.128 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:38.128 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:38.133 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:38.133 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:38.133 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.133 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:29:38.133 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:38.134 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:38.134 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:29:38.134 INFO:tasks.workunit.client.0.smithi148.stdout:read: 512 2024-03-07T03:29:38.134 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.134 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:38.134 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:38.135 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:38.136 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:29:38.137 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:29:38.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:38 smithi195 bash[66542]: cluster 2024-03-07T03:29:37.317273+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2024-03-07T03:29:38.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:38 smithi195 bash[66542]: audit 2024-03-07T03:29:37.912013+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:38.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:38 smithi148 bash[79819]: cluster 2024-03-07T03:29:37.317273+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2024-03-07T03:29:38.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:38 smithi148 bash[79819]: audit 2024-03-07T03:29:37.912013+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:38 smithi148 bash[82222]: cluster 2024-03-07T03:29:37.317273+0000 mon.a (mon.0) 1140 : cluster 0 osdmap e678: 8 total, 8 up, 8 in 2024-03-07T03:29:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:38 smithi148 bash[82222]: audit 2024-03-07T03:29:37.912013+0000 mon.a (mon.0) 1141 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:40.362 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:29:40.362 INFO:tasks.workunit.client.0.smithi148.stdout:finished read 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 0 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 16384 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.363 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 49152 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 65536 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.364 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 98304 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 114688 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 147456 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.365 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 163840 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 196608 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.366 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 212992 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.367 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.368 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.369 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.369 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.369 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.369 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.369 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.369 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.373 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.373 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.373 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.374 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.374 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.374 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.374 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.385 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.385 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.386 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.410 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.410 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.410 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.410 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.410 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.410 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.410 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.439 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.439 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.439 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.439 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.439 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.439 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.439 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.478 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.479 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.479 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.479 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.479 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.479 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.479 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.518 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.519 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.519 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.519 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.519 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.519 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:40 smithi195 bash[66542]: cluster 2024-03-07T03:29:38.678656+0000 mgr.y (mgr.34101) 471 : cluster 0 pgmap v768: 457 pgs: 457 active+clean; 273 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 8.7 MiB/s wr, 710 op/s 2024-03-07T03:29:40.590 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.590 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.590 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.590 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.590 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.590 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.590 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.636 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.636 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.636 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.636 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.636 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.636 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.636 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.693 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.694 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.694 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.694 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.694 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.694 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.694 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.753 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.753 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.753 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.753 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.753 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.754 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:40 smithi148 bash[79819]: cluster 2024-03-07T03:29:38.678656+0000 mgr.y (mgr.34101) 471 : cluster 0 pgmap v768: 457 pgs: 457 active+clean; 273 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 8.7 MiB/s wr, 710 op/s 2024-03-07T03:29:40.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:40 smithi148 bash[82222]: cluster 2024-03-07T03:29:38.678656+0000 mgr.y (mgr.34101) 471 : cluster 0 pgmap v768: 457 pgs: 457 active+clean; 273 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 8.7 MiB/s wr, 710 op/s 2024-03-07T03:29:40.831 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.832 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.832 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.832 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.832 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.832 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.832 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.909 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.909 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.909 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.909 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.909 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.909 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.910 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:40.991 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:40.991 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:40.991 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:40.991 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:40.991 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:40.992 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:40.992 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:41.076 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:41.076 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:41.076 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:41.076 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.077 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:41.181 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:41.181 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:41.181 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:41.181 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:41.182 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.182 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.182 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:41.276 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:41.276 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:41.276 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:41.276 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:41.276 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.276 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.276 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:41.390 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:41.391 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:41.391 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:41.391 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:41.391 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.391 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.391 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:29:41.489 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:finished writesame 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:to verify the data 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:verified 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:started writesame 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:expected fail, finished writesame 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (3432 ms) 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-03-07T03:29:41.490 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:42.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: cluster 2024-03-07T03:29:40.680209+0000 mgr.y (mgr.34101) 472 : cluster 0 pgmap v769: 457 pgs: 457 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-03-07T03:29:42.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: audit 2024-03-07T03:29:40.797179+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: audit 2024-03-07T03:29:40.803583+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: audit 2024-03-07T03:29:41.000376+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: audit 2024-03-07T03:29:41.008797+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: audit 2024-03-07T03:29:41.591802+0000 mon.a (mon.0) 1146 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: audit 2024-03-07T03:29:41.593463+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[79819]: audit 2024-03-07T03:29:41.601840+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: cluster 2024-03-07T03:29:40.680209+0000 mgr.y (mgr.34101) 472 : cluster 0 pgmap v769: 457 pgs: 457 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: audit 2024-03-07T03:29:40.797179+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: audit 2024-03-07T03:29:40.803583+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: audit 2024-03-07T03:29:41.000376+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: audit 2024-03-07T03:29:41.008797+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: audit 2024-03-07T03:29:41.591802+0000 mon.a (mon.0) 1146 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:42.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: audit 2024-03-07T03:29:41.593463+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:42.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:41 smithi148 bash[82222]: audit 2024-03-07T03:29:41.601840+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: cluster 2024-03-07T03:29:40.680209+0000 mgr.y (mgr.34101) 472 : cluster 0 pgmap v769: 457 pgs: 457 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.07k op/s 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: audit 2024-03-07T03:29:40.797179+0000 mon.a (mon.0) 1142 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: audit 2024-03-07T03:29:40.803583+0000 mon.a (mon.0) 1143 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: audit 2024-03-07T03:29:41.000376+0000 mon.a (mon.0) 1144 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: audit 2024-03-07T03:29:41.008797+0000 mon.a (mon.0) 1145 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: audit 2024-03-07T03:29:41.591802+0000 mon.a (mon.0) 1146 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: audit 2024-03-07T03:29:41.593463+0000 mon.a (mon.0) 1147 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:29:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:41 smithi195 bash[66542]: audit 2024-03-07T03:29:41.601840+0000 mon.a (mon.0) 1148 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:29:42 smithi195 bash[59134]: ts=2024-03-07T03:29:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:43.432 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 0 2024-03-07T03:29:43.432 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:29:43.432 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:29:43.432 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:snap: orig 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:found orig with size 2097152 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:read: 80 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:write test data! 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 80 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 2 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:snap: orig 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:snap: written 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:found orig with size 2097152 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:found written with size 2097152 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:read: 80 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:read: 80 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:read: 80 2024-03-07T03:29:43.433 INFO:tasks.workunit.client.0.smithi148.stdout:write to snapshot returned -30 2024-03-07T03:29:43.434 INFO:tasks.workunit.client.0.smithi148.stdout:Read-only file system 2024-03-07T03:29:43.497 INFO:tasks.workunit.client.0.smithi148.stdout:read: 80 2024-03-07T03:29:43.497 INFO:tasks.workunit.client.0.smithi148.stdout:read: 80 2024-03-07T03:29:43.498 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 80 2024-03-07T03:29:43.498 INFO:tasks.workunit.client.0.smithi148.stdout:opening testimg@orig 2024-03-07T03:29:43.498 INFO:tasks.workunit.client.0.smithi148.stdout:read: 80 2024-03-07T03:29:43.498 INFO:tasks.workunit.client.0.smithi148.stdout:write to snapshot returned -30 2024-03-07T03:29:43.498 INFO:tasks.workunit.client.0.smithi148.stdout:Read-only file system 2024-03-07T03:29:43.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:43 smithi195 bash[66542]: cluster 2024-03-07T03:29:42.324469+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2024-03-07T03:29:43.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:43 smithi195 bash[66542]: audit 2024-03-07T03:29:42.337157+0000 mon.a (mon.0) 1150 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:43.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:43 smithi195 bash[66542]: audit 2024-03-07T03:29:42.340278+0000 mon.a (mon.0) 1151 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:43 smithi148 bash[82222]: cluster 2024-03-07T03:29:42.324469+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2024-03-07T03:29:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:43 smithi148 bash[82222]: audit 2024-03-07T03:29:42.337157+0000 mon.a (mon.0) 1150 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:43.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:43 smithi148 bash[82222]: audit 2024-03-07T03:29:42.340278+0000 mon.a (mon.0) 1151 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:43.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:43 smithi148 bash[79819]: cluster 2024-03-07T03:29:42.324469+0000 mon.a (mon.0) 1149 : cluster 0 osdmap e679: 8 total, 8 up, 8 in 2024-03-07T03:29:43.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:43 smithi148 bash[79819]: audit 2024-03-07T03:29:42.337157+0000 mon.a (mon.0) 1150 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:29:43.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:43 smithi148 bash[79819]: audit 2024-03-07T03:29:42.340278+0000 mon.a (mon.0) 1151 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:44 smithi148 bash[79819]: cluster 2024-03-07T03:29:42.681061+0000 mgr.y (mgr.34101) 473 : cluster 0 pgmap v771: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.53k op/s 2024-03-07T03:29:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:44 smithi148 bash[79819]: cluster 2024-03-07T03:29:43.346896+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2024-03-07T03:29:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:44 smithi148 bash[82222]: cluster 2024-03-07T03:29:42.681061+0000 mgr.y (mgr.34101) 473 : cluster 0 pgmap v771: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.53k op/s 2024-03-07T03:29:44.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:44 smithi148 bash[82222]: cluster 2024-03-07T03:29:43.346896+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2024-03-07T03:29:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:44 smithi195 bash[66542]: cluster 2024-03-07T03:29:42.681061+0000 mgr.y (mgr.34101) 473 : cluster 0 pgmap v771: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 7.3 MiB/s wr, 1.53k op/s 2024-03-07T03:29:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:44 smithi195 bash[66542]: cluster 2024-03-07T03:29:43.346896+0000 mon.a (mon.0) 1152 : cluster 0 osdmap e680: 8 total, 8 up, 8 in 2024-03-07T03:29:45.429 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 2 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:snap: orig 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:snap: written 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:found orig with size 2097152 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:found written with size 2097152 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 1 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:snap: orig 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:found orig with size 2097152 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 0 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (3940 ms) 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-03-07T03:29:45.430 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:45.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:45 smithi148 bash[79819]: cluster 2024-03-07T03:29:44.360358+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2024-03-07T03:29:45.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:45 smithi148 bash[82222]: cluster 2024-03-07T03:29:44.360358+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2024-03-07T03:29:45.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:45 smithi195 bash[66542]: cluster 2024-03-07T03:29:44.360358+0000 mon.a (mon.0) 1153 : cluster 0 osdmap e681: 8 total, 8 up, 8 in 2024-03-07T03:29:46.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:46 smithi148 bash[79819]: cluster 2024-03-07T03:29:44.682015+0000 mgr.y (mgr.34101) 474 : cluster 0 pgmap v774: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 5.1 MiB/s wr, 1.94k op/s 2024-03-07T03:29:46.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:46 smithi148 bash[79819]: cluster 2024-03-07T03:29:45.370655+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2024-03-07T03:29:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:46 smithi148 bash[82222]: cluster 2024-03-07T03:29:44.682015+0000 mgr.y (mgr.34101) 474 : cluster 0 pgmap v774: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 5.1 MiB/s wr, 1.94k op/s 2024-03-07T03:29:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:46 smithi148 bash[82222]: cluster 2024-03-07T03:29:45.370655+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2024-03-07T03:29:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:46 smithi195 bash[66542]: cluster 2024-03-07T03:29:44.682015+0000 mgr.y (mgr.34101) 474 : cluster 0 pgmap v774: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 5.6 MiB/s rd, 5.1 MiB/s wr, 1.94k op/s 2024-03-07T03:29:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:46 smithi195 bash[66542]: cluster 2024-03-07T03:29:45.370655+0000 mon.a (mon.0) 1154 : cluster 0 osdmap e682: 8 total, 8 up, 8 in 2024-03-07T03:29:47.441 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2012 ms) 2024-03-07T03:29:47.441 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestClone 2024-03-07T03:29:47.442 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:47.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:47 smithi148 bash[79819]: cluster 2024-03-07T03:29:46.373041+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2024-03-07T03:29:47.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:29:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:29:47] "GET /metrics HTTP/1.1" 200 63136 "" "Prometheus/2.43.0" 2024-03-07T03:29:47.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:47 smithi148 bash[82222]: cluster 2024-03-07T03:29:46.373041+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2024-03-07T03:29:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:47 smithi195 bash[66542]: cluster 2024-03-07T03:29:46.373041+0000 mon.a (mon.0) 1155 : cluster 0 osdmap e683: 8 total, 8 up, 8 in 2024-03-07T03:29:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:48 smithi148 bash[79819]: cluster 2024-03-07T03:29:46.682991+0000 mgr.y (mgr.34101) 475 : cluster 0 pgmap v777: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 137 KiB/s rd, 56 KiB/s wr, 276 op/s 2024-03-07T03:29:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:48 smithi148 bash[79819]: cluster 2024-03-07T03:29:47.392141+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2024-03-07T03:29:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:48 smithi148 bash[82222]: cluster 2024-03-07T03:29:46.682991+0000 mgr.y (mgr.34101) 475 : cluster 0 pgmap v777: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 137 KiB/s rd, 56 KiB/s wr, 276 op/s 2024-03-07T03:29:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:48 smithi148 bash[82222]: cluster 2024-03-07T03:29:47.392141+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2024-03-07T03:29:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:48 smithi195 bash[66542]: cluster 2024-03-07T03:29:46.682991+0000 mgr.y (mgr.34101) 475 : cluster 0 pgmap v777: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 137 KiB/s rd, 56 KiB/s wr, 276 op/s 2024-03-07T03:29:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:48 smithi195 bash[66542]: cluster 2024-03-07T03:29:47.392141+0000 mon.a (mon.0) 1156 : cluster 0 osdmap e684: 8 total, 8 up, 8 in 2024-03-07T03:29:49.446 INFO:tasks.workunit.client.0.smithi148.stdout:made parent image "parent" 2024-03-07T03:29:49.446 INFO:tasks.workunit.client.0.smithi148.stdout:parent has no parent info 2024-03-07T03:29:49.446 INFO:tasks.workunit.client.0.smithi148.stdout:made snapshot "parent@parent_snap" 2024-03-07T03:29:49.446 INFO:tasks.workunit.client.0.smithi148.stdout:can't unprotect an unprotected snap 2024-03-07T03:29:49.446 INFO:tasks.workunit.client.0.smithi148.stdout:can't protect a protected snap 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:made and opened clone "child" 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:read: 8 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:read: 8 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:read: 8 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:sizes and overlaps are good between parent and child 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:child image successfully cloned all image-meta pairs 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:sized down clone, changed overlap 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:sized up clone, changed size but not overlap or parent's size 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:can't remove parent while child still exists 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:can't remove parent while still protected 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:removed parent snap after unprotecting 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestClone (2005 ms) 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestClone2 2024-03-07T03:29:49.447 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:49.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:49 smithi148 bash[79819]: cluster 2024-03-07T03:29:48.403899+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2024-03-07T03:29:49.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:49 smithi148 bash[82222]: cluster 2024-03-07T03:29:48.403899+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2024-03-07T03:29:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:49 smithi195 bash[66542]: cluster 2024-03-07T03:29:48.403899+0000 mon.a (mon.0) 1157 : cluster 0 osdmap e685: 8 total, 8 up, 8 in 2024-03-07T03:29:50.746 INFO:tasks.workunit.client.0.smithi148.stdout:made parent image "parent" 2024-03-07T03:29:50.746 INFO:tasks.workunit.client.0.smithi148.stdout:parent has no parent info 2024-03-07T03:29:50.746 INFO:tasks.workunit.client.0.smithi148.stdout:made snapshot "parent@parent_snap" 2024-03-07T03:29:50.746 INFO:tasks.workunit.client.0.smithi148.stdout:made and opened clone "child" 2024-03-07T03:29:50.746 INFO:tasks.workunit.client.0.smithi148.stdout:child image successfully cloned all image-meta pairs 2024-03-07T03:29:50.747 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestClone2 (1299 ms) 2024-03-07T03:29:50.747 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ListChildren 2024-03-07T03:29:50.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:50 smithi148 bash[79819]: cluster 2024-03-07T03:29:48.684915+0000 mgr.y (mgr.34101) 476 : cluster 0 pgmap v780: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 146 KiB/s rd, 57 KiB/s wr, 289 op/s 2024-03-07T03:29:50.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:50 smithi148 bash[79819]: cluster 2024-03-07T03:29:49.417937+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2024-03-07T03:29:50.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:50 smithi148 bash[82222]: cluster 2024-03-07T03:29:48.684915+0000 mgr.y (mgr.34101) 476 : cluster 0 pgmap v780: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 146 KiB/s rd, 57 KiB/s wr, 289 op/s 2024-03-07T03:29:50.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:50 smithi148 bash[82222]: cluster 2024-03-07T03:29:49.417937+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2024-03-07T03:29:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:50 smithi195 bash[66542]: cluster 2024-03-07T03:29:48.684915+0000 mgr.y (mgr.34101) 476 : cluster 0 pgmap v780: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 146 KiB/s rd, 57 KiB/s wr, 289 op/s 2024-03-07T03:29:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:50 smithi195 bash[66542]: cluster 2024-03-07T03:29:49.417937+0000 mon.a (mon.0) 1158 : cluster 0 osdmap e686: 8 total, 8 up, 8 in 2024-03-07T03:29:51.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:51 smithi148 bash[79819]: cluster 2024-03-07T03:29:50.416505+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2024-03-07T03:29:51.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:51 smithi148 bash[82222]: cluster 2024-03-07T03:29:50.416505+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2024-03-07T03:29:51.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:51 smithi195 bash[66542]: cluster 2024-03-07T03:29:50.416505+0000 mon.a (mon.0) 1159 : cluster 0 osdmap e687: 8 total, 8 up, 8 in 2024-03-07T03:29:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:29:52 smithi195 bash[59134]: ts=2024-03-07T03:29:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:29:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:52 smithi195 bash[66542]: cluster 2024-03-07T03:29:50.685897+0000 mgr.y (mgr.34101) 477 : cluster 0 pgmap v783: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 9.7 KiB/s wr, 111 op/s 2024-03-07T03:29:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:52 smithi195 bash[66542]: cluster 2024-03-07T03:29:51.441109+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2024-03-07T03:29:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:52 smithi195 bash[66542]: audit 2024-03-07T03:29:51.443123+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.148:0/4280129078' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:53.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:52 smithi195 bash[66542]: audit 2024-03-07T03:29:51.444244+0000 mon.a (mon.0) 1161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:53.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:52 smithi195 bash[66542]: audit 2024-03-07T03:29:52.311025+0000 mon.a (mon.0) 1162 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:53.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:52 smithi195 bash[66542]: cluster 2024-03-07T03:29:52.319588+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2024-03-07T03:29:53.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[79819]: cluster 2024-03-07T03:29:50.685897+0000 mgr.y (mgr.34101) 477 : cluster 0 pgmap v783: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 9.7 KiB/s wr, 111 op/s 2024-03-07T03:29:53.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[79819]: cluster 2024-03-07T03:29:51.441109+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2024-03-07T03:29:53.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[79819]: audit 2024-03-07T03:29:51.443123+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.148:0/4280129078' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:53.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[79819]: audit 2024-03-07T03:29:51.444244+0000 mon.a (mon.0) 1161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:53.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[79819]: audit 2024-03-07T03:29:52.311025+0000 mon.a (mon.0) 1162 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:53.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[79819]: cluster 2024-03-07T03:29:52.319588+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2024-03-07T03:29:53.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[82222]: cluster 2024-03-07T03:29:50.685897+0000 mgr.y (mgr.34101) 477 : cluster 0 pgmap v783: 457 pgs: 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 9.7 KiB/s wr, 111 op/s 2024-03-07T03:29:53.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[82222]: cluster 2024-03-07T03:29:51.441109+0000 mon.a (mon.0) 1160 : cluster 0 osdmap e688: 8 total, 8 up, 8 in 2024-03-07T03:29:53.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[82222]: audit 2024-03-07T03:29:51.443123+0000 mon.c (mon.2) 61 : audit 1 from='client.? 172.21.15.148:0/4280129078' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:53.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[82222]: audit 2024-03-07T03:29:51.444244+0000 mon.a (mon.0) 1161 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:53.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[82222]: audit 2024-03-07T03:29:52.311025+0000 mon.a (mon.0) 1162 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:53.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:52 smithi148 bash[82222]: cluster 2024-03-07T03:29:52.319588+0000 mon.a (mon.0) 1163 : cluster 0 osdmap e689: 8 total, 8 up, 8 in 2024-03-07T03:29:54.334 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:29:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[79819]: cluster 2024-03-07T03:29:52.686908+0000 mgr.y (mgr.34101) 478 : cluster 0 pgmap v786: 489 pgs: 32 unknown, 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 427 KiB/s rd, 29 KiB/s wr, 514 op/s 2024-03-07T03:29:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[79819]: cluster 2024-03-07T03:29:53.328798+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2024-03-07T03:29:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[79819]: audit 2024-03-07T03:29:53.336622+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.148:0/1416769896' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[79819]: audit 2024-03-07T03:29:53.344585+0000 mon.a (mon.0) 1165 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[82222]: cluster 2024-03-07T03:29:52.686908+0000 mgr.y (mgr.34101) 478 : cluster 0 pgmap v786: 489 pgs: 32 unknown, 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 427 KiB/s rd, 29 KiB/s wr, 514 op/s 2024-03-07T03:29:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[82222]: cluster 2024-03-07T03:29:53.328798+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2024-03-07T03:29:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[82222]: audit 2024-03-07T03:29:53.336622+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.148:0/1416769896' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:54 smithi148 bash[82222]: audit 2024-03-07T03:29:53.344585+0000 mon.a (mon.0) 1165 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:54 smithi195 bash[66542]: cluster 2024-03-07T03:29:52.686908+0000 mgr.y (mgr.34101) 478 : cluster 0 pgmap v786: 489 pgs: 32 unknown, 457 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 427 KiB/s rd, 29 KiB/s wr, 514 op/s 2024-03-07T03:29:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:54 smithi195 bash[66542]: cluster 2024-03-07T03:29:53.328798+0000 mon.a (mon.0) 1164 : cluster 0 osdmap e690: 8 total, 8 up, 8 in 2024-03-07T03:29:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:54 smithi195 bash[66542]: audit 2024-03-07T03:29:53.336622+0000 mon.c (mon.2) 62 : audit 1 from='client.? 172.21.15.148:0/1416769896' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:54 smithi195 bash[66542]: audit 2024-03-07T03:29:53.344585+0000 mon.a (mon.0) 1165 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:29:55.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:55 smithi148 bash[79819]: audit 2024-03-07T03:29:54.325699+0000 mon.a (mon.0) 1166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:55.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:55 smithi148 bash[79819]: cluster 2024-03-07T03:29:54.333093+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2024-03-07T03:29:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:55 smithi148 bash[82222]: audit 2024-03-07T03:29:54.325699+0000 mon.a (mon.0) 1166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:55 smithi148 bash[82222]: cluster 2024-03-07T03:29:54.333093+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2024-03-07T03:29:55.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:55 smithi195 bash[66542]: audit 2024-03-07T03:29:54.325699+0000 mon.a (mon.0) 1166 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:29:55.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:55 smithi195 bash[66542]: cluster 2024-03-07T03:29:54.333093+0000 mon.a (mon.0) 1167 : cluster 0 osdmap e691: 8 total, 8 up, 8 in 2024-03-07T03:29:56.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:56 smithi148 bash[79819]: cluster 2024-03-07T03:29:54.687945+0000 mgr.y (mgr.34101) 479 : cluster 0 pgmap v789: 521 pgs: 19 creating+peering, 26 unknown, 476 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 745 KiB/s rd, 50 KiB/s wr, 877 op/s 2024-03-07T03:29:56.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:56 smithi148 bash[79819]: cluster 2024-03-07T03:29:55.359652+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2024-03-07T03:29:56.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:56 smithi148 bash[82222]: cluster 2024-03-07T03:29:54.687945+0000 mgr.y (mgr.34101) 479 : cluster 0 pgmap v789: 521 pgs: 19 creating+peering, 26 unknown, 476 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 745 KiB/s rd, 50 KiB/s wr, 877 op/s 2024-03-07T03:29:56.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:56 smithi148 bash[82222]: cluster 2024-03-07T03:29:55.359652+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2024-03-07T03:29:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:56 smithi195 bash[66542]: cluster 2024-03-07T03:29:54.687945+0000 mgr.y (mgr.34101) 479 : cluster 0 pgmap v789: 521 pgs: 19 creating+peering, 26 unknown, 476 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 745 KiB/s rd, 50 KiB/s wr, 877 op/s 2024-03-07T03:29:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:56 smithi195 bash[66542]: cluster 2024-03-07T03:29:55.359652+0000 mon.a (mon.0) 1168 : cluster 0 osdmap e692: 8 total, 8 up, 8 in 2024-03-07T03:29:57.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:29:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:29:57] "GET /metrics HTTP/1.1" 200 63136 "" "Prometheus/2.43.0" 2024-03-07T03:29:57.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:57 smithi148 bash[79819]: cluster 2024-03-07T03:29:56.365087+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2024-03-07T03:29:57.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:57 smithi148 bash[79819]: cluster 2024-03-07T03:29:57.319291+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2024-03-07T03:29:57.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:57 smithi148 bash[79819]: audit 2024-03-07T03:29:57.323933+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:57.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:57 smithi148 bash[82222]: cluster 2024-03-07T03:29:56.365087+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2024-03-07T03:29:57.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:57 smithi148 bash[82222]: cluster 2024-03-07T03:29:57.319291+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2024-03-07T03:29:57.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:57 smithi148 bash[82222]: audit 2024-03-07T03:29:57.323933+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:57 smithi195 bash[66542]: cluster 2024-03-07T03:29:56.365087+0000 mon.a (mon.0) 1169 : cluster 0 osdmap e693: 8 total, 8 up, 8 in 2024-03-07T03:29:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:57 smithi195 bash[66542]: cluster 2024-03-07T03:29:57.319291+0000 mon.a (mon.0) 1170 : cluster 0 osdmap e694: 8 total, 8 up, 8 in 2024-03-07T03:29:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:57 smithi195 bash[66542]: audit 2024-03-07T03:29:57.323933+0000 mon.a (mon.0) 1171 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:29:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:58 smithi148 bash[79819]: cluster 2024-03-07T03:29:56.689022+0000 mgr.y (mgr.34101) 480 : cluster 0 pgmap v792: 521 pgs: 19 creating+peering, 502 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 358 KiB/s rd, 26 KiB/s wr, 424 op/s 2024-03-07T03:29:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:58 smithi148 bash[82222]: cluster 2024-03-07T03:29:56.689022+0000 mgr.y (mgr.34101) 480 : cluster 0 pgmap v792: 521 pgs: 19 creating+peering, 502 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 358 KiB/s rd, 26 KiB/s wr, 424 op/s 2024-03-07T03:29:58.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:58 smithi195 bash[66542]: cluster 2024-03-07T03:29:56.689022+0000 mgr.y (mgr.34101) 480 : cluster 0 pgmap v792: 521 pgs: 19 creating+peering, 502 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 358 KiB/s rd, 26 KiB/s wr, 424 op/s 2024-03-07T03:29:59.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:29:59 smithi148 bash[79819]: cluster 2024-03-07T03:29:58.400807+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2024-03-07T03:29:59.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:29:59 smithi148 bash[82222]: cluster 2024-03-07T03:29:58.400807+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2024-03-07T03:29:59.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:29:59 smithi195 bash[66542]: cluster 2024-03-07T03:29:58.400807+0000 mon.a (mon.0) 1172 : cluster 0 osdmap e695: 8 total, 8 up, 8 in 2024-03-07T03:30:00.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:00 smithi148 bash[79819]: debug 2024-03-07T03:29:59.993+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:00.735 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.735 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.735 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.735 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.735 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 1 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:child: image113 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113/acb84c552731 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.736 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 2 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:child: image113 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113/acb84c552731 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.737 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.738 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 3 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:child: image113 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:child: image115 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113/acb84c552731 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115/acb87eca7c48 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.739 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 3 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:child: image113 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:child: image115 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113/acb84c552731 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.740 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115/acb87eca7c48 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.741 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 4 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 4 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:child: image113 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:child: image115 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:child: image116 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113/acb84c552731 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115/acb87eca7c48 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116/acb83b906289 2024-03-07T03:30:00.742 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:00.743 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:00.744 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:00.744 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.744 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:00.744 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:00.744 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:00.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:00 smithi148 bash[79819]: cluster 2024-03-07T03:29:58.691440+0000 mgr.y (mgr.34101) 481 : cluster 0 pgmap v795: 521 pgs: 10 creating+peering, 511 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 255 B/s wr, 1 op/s 2024-03-07T03:30:00.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:00 smithi148 bash[79819]: cluster 2024-03-07T03:29:59.418661+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2024-03-07T03:30:00.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:00 smithi148 bash[79819]: cluster 2024-03-07T03:30:00.000174+0000 mon.a (mon.0) 1174 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:00 smithi148 bash[82222]: cluster 2024-03-07T03:29:58.691440+0000 mgr.y (mgr.34101) 481 : cluster 0 pgmap v795: 521 pgs: 10 creating+peering, 511 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 255 B/s wr, 1 op/s 2024-03-07T03:30:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:00 smithi148 bash[82222]: cluster 2024-03-07T03:29:59.418661+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2024-03-07T03:30:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:00 smithi148 bash[82222]: cluster 2024-03-07T03:30:00.000174+0000 mon.a (mon.0) 1174 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:00.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:00 smithi195 bash[66542]: cluster 2024-03-07T03:29:58.691440+0000 mgr.y (mgr.34101) 481 : cluster 0 pgmap v795: 521 pgs: 10 creating+peering, 511 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 255 B/s wr, 1 op/s 2024-03-07T03:30:00.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:00 smithi195 bash[66542]: cluster 2024-03-07T03:29:59.418661+0000 mon.a (mon.0) 1173 : cluster 0 osdmap e696: 8 total, 8 up, 8 in 2024-03-07T03:30:00.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:00 smithi195 bash[66542]: cluster 2024-03-07T03:30:00.000174+0000 mon.a (mon.0) 1174 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:30:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:02 smithi195 bash[66542]: cluster 2024-03-07T03:30:00.693471+0000 mgr.y (mgr.34101) 482 : cluster 0 pgmap v797: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.9 KiB/s wr, 77 op/s 2024-03-07T03:30:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:30:02 smithi195 bash[59134]: ts=2024-03-07T03:30:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:02.611 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image113 2024-03-07T03:30:02.611 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:02.611 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.611 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.611 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 4 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 4 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:child: image113 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:child: image115 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:child: image116 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image113/acb84c552731 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115/acb87eca7c48 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.612 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116/acb83b906289 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.613 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image115 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 3 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:child: image115 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:child: image116 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.614 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image115/acb87eca7c48 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116/acb83b906289 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.615 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 2 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:child: image116 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-28/image116/acb83b906289 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.616 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 1 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:child: image114 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-27/image114/acb86f2c5323 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 0 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ListChildren (11866 ms) 2024-03-07T03:30:02.617 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-03-07T03:30:02.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:02 smithi148 bash[79819]: cluster 2024-03-07T03:30:00.693471+0000 mgr.y (mgr.34101) 482 : cluster 0 pgmap v797: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.9 KiB/s wr, 77 op/s 2024-03-07T03:30:02.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:02 smithi148 bash[82222]: cluster 2024-03-07T03:30:00.693471+0000 mgr.y (mgr.34101) 482 : cluster 0 pgmap v797: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 58 KiB/s rd, 3.9 KiB/s wr, 77 op/s 2024-03-07T03:30:03.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:03 smithi148 bash[79819]: cluster 2024-03-07T03:30:02.426671+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2024-03-07T03:30:03.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:03 smithi148 bash[82222]: cluster 2024-03-07T03:30:02.426671+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2024-03-07T03:30:03.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:03 smithi195 bash[66542]: cluster 2024-03-07T03:30:02.426671+0000 mon.a (mon.0) 1175 : cluster 0 osdmap e697: 8 total, 8 up, 8 in 2024-03-07T03:30:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[79819]: cluster 2024-03-07T03:30:02.694102+0000 mgr.y (mgr.34101) 483 : cluster 0 pgmap v799: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 198 KiB/s rd, 5.0 KiB/s wr, 239 op/s 2024-03-07T03:30:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[79819]: cluster 2024-03-07T03:30:03.446894+0000 mon.a (mon.0) 1176 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2024-03-07T03:30:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[79819]: audit 2024-03-07T03:30:03.456769+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.148:0/972168694' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[79819]: audit 2024-03-07T03:30:03.457430+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:04.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[82222]: cluster 2024-03-07T03:30:02.694102+0000 mgr.y (mgr.34101) 483 : cluster 0 pgmap v799: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 198 KiB/s rd, 5.0 KiB/s wr, 239 op/s 2024-03-07T03:30:04.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[82222]: cluster 2024-03-07T03:30:03.446894+0000 mon.a (mon.0) 1176 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2024-03-07T03:30:04.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[82222]: audit 2024-03-07T03:30:03.456769+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.148:0/972168694' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:04.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:04 smithi148 bash[82222]: audit 2024-03-07T03:30:03.457430+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:04 smithi195 bash[66542]: cluster 2024-03-07T03:30:02.694102+0000 mgr.y (mgr.34101) 483 : cluster 0 pgmap v799: 521 pgs: 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 198 KiB/s rd, 5.0 KiB/s wr, 239 op/s 2024-03-07T03:30:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:04 smithi195 bash[66542]: cluster 2024-03-07T03:30:03.446894+0000 mon.a (mon.0) 1176 : cluster 0 osdmap e698: 8 total, 8 up, 8 in 2024-03-07T03:30:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:04 smithi195 bash[66542]: audit 2024-03-07T03:30:03.456769+0000 mon.c (mon.2) 63 : audit 1 from='client.? 172.21.15.148:0/972168694' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:04 smithi195 bash[66542]: audit 2024-03-07T03:30:03.457430+0000 mon.a (mon.0) 1177 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:05.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:05 smithi148 bash[79819]: audit 2024-03-07T03:30:04.435985+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:05.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:05 smithi148 bash[79819]: cluster 2024-03-07T03:30:04.443813+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2024-03-07T03:30:05.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:05 smithi148 bash[82222]: audit 2024-03-07T03:30:04.435985+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:05.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:05 smithi148 bash[82222]: cluster 2024-03-07T03:30:04.443813+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2024-03-07T03:30:05.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:05 smithi195 bash[66542]: audit 2024-03-07T03:30:04.435985+0000 mon.a (mon.0) 1178 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:05.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:05 smithi195 bash[66542]: cluster 2024-03-07T03:30:04.443813+0000 mon.a (mon.0) 1179 : cluster 0 osdmap e699: 8 total, 8 up, 8 in 2024-03-07T03:30:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:06 smithi148 bash[82222]: cluster 2024-03-07T03:30:04.695053+0000 mgr.y (mgr.34101) 484 : cluster 0 pgmap v802: 553 pgs: 9 creating+peering, 23 unknown, 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 315 KiB/s rd, 8.3 KiB/s wr, 385 op/s 2024-03-07T03:30:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:06 smithi148 bash[82222]: cluster 2024-03-07T03:30:05.471122+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2024-03-07T03:30:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:06 smithi148 bash[82222]: audit 2024-03-07T03:30:05.480184+0000 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.148:0/518079514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:06 smithi148 bash[79819]: cluster 2024-03-07T03:30:04.695053+0000 mgr.y (mgr.34101) 484 : cluster 0 pgmap v802: 553 pgs: 9 creating+peering, 23 unknown, 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 315 KiB/s rd, 8.3 KiB/s wr, 385 op/s 2024-03-07T03:30:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:06 smithi148 bash[79819]: cluster 2024-03-07T03:30:05.471122+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2024-03-07T03:30:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:06 smithi148 bash[79819]: audit 2024-03-07T03:30:05.480184+0000 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.148:0/518079514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:06 smithi195 bash[66542]: cluster 2024-03-07T03:30:04.695053+0000 mgr.y (mgr.34101) 484 : cluster 0 pgmap v802: 553 pgs: 9 creating+peering, 23 unknown, 521 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 315 KiB/s rd, 8.3 KiB/s wr, 385 op/s 2024-03-07T03:30:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:06 smithi195 bash[66542]: cluster 2024-03-07T03:30:05.471122+0000 mon.a (mon.0) 1180 : cluster 0 osdmap e700: 8 total, 8 up, 8 in 2024-03-07T03:30:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:06 smithi195 bash[66542]: audit 2024-03-07T03:30:05.480184+0000 mon.a (mon.0) 1181 : audit 1 from='client.? 172.21.15.148:0/518079514' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-30","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:07.478 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:30:07] "GET /metrics HTTP/1.1" 200 67867 "" "Prometheus/2.43.0" 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[79819]: audit 2024-03-07T03:30:06.464181+0000 mon.a (mon.0) 1182 : audit 1 from='client.? 172.21.15.148:0/518079514' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[79819]: cluster 2024-03-07T03:30:06.472961+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[79819]: cluster 2024-03-07T03:30:07.326950+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[79819]: audit 2024-03-07T03:30:07.331295+0000 mon.c (mon.2) 64 : audit 1 from='client.? 172.21.15.148:0/3008139526' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[79819]: audit 2024-03-07T03:30:07.332027+0000 mon.a (mon.0) 1185 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[82222]: audit 2024-03-07T03:30:06.464181+0000 mon.a (mon.0) 1182 : audit 1 from='client.? 172.21.15.148:0/518079514' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[82222]: cluster 2024-03-07T03:30:06.472961+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2024-03-07T03:30:07.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[82222]: cluster 2024-03-07T03:30:07.326950+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2024-03-07T03:30:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[82222]: audit 2024-03-07T03:30:07.331295+0000 mon.c (mon.2) 64 : audit 1 from='client.? 172.21.15.148:0/3008139526' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:07.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:07 smithi148 bash[82222]: audit 2024-03-07T03:30:07.332027+0000 mon.a (mon.0) 1185 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:07 smithi195 bash[66542]: audit 2024-03-07T03:30:06.464181+0000 mon.a (mon.0) 1182 : audit 1 from='client.? 172.21.15.148:0/518079514' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:07 smithi195 bash[66542]: cluster 2024-03-07T03:30:06.472961+0000 mon.a (mon.0) 1183 : cluster 0 osdmap e701: 8 total, 8 up, 8 in 2024-03-07T03:30:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:07 smithi195 bash[66542]: cluster 2024-03-07T03:30:07.326950+0000 mon.a (mon.0) 1184 : cluster 0 osdmap e702: 8 total, 8 up, 8 in 2024-03-07T03:30:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:07 smithi195 bash[66542]: audit 2024-03-07T03:30:07.331295+0000 mon.c (mon.2) 64 : audit 1 from='client.? 172.21.15.148:0/3008139526' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:07 smithi195 bash[66542]: audit 2024-03-07T03:30:07.332027+0000 mon.a (mon.0) 1185 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:30:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[82222]: cluster 2024-03-07T03:30:06.696098+0000 mgr.y (mgr.34101) 485 : cluster 0 pgmap v805: 585 pgs: 12 creating+peering, 37 unknown, 536 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 329 KiB/s rd, 9.5 KiB/s wr, 407 op/s 2024-03-07T03:30:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[82222]: audit 2024-03-07T03:30:07.978197+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[82222]: audit 2024-03-07T03:30:08.322637+0000 mon.a (mon.0) 1187 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[82222]: cluster 2024-03-07T03:30:08.330675+0000 mon.a (mon.0) 1188 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2024-03-07T03:30:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[82222]: audit 2024-03-07T03:30:08.337793+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]: dispatch 2024-03-07T03:30:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[82222]: audit 2024-03-07T03:30:08.343505+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]: dispatch 2024-03-07T03:30:08.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[79819]: cluster 2024-03-07T03:30:06.696098+0000 mgr.y (mgr.34101) 485 : cluster 0 pgmap v805: 585 pgs: 12 creating+peering, 37 unknown, 536 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 329 KiB/s rd, 9.5 KiB/s wr, 407 op/s 2024-03-07T03:30:08.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[79819]: audit 2024-03-07T03:30:07.978197+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:08.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[79819]: audit 2024-03-07T03:30:08.322637+0000 mon.a (mon.0) 1187 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:08.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[79819]: cluster 2024-03-07T03:30:08.330675+0000 mon.a (mon.0) 1188 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2024-03-07T03:30:08.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[79819]: audit 2024-03-07T03:30:08.337793+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]: dispatch 2024-03-07T03:30:08.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:08 smithi148 bash[79819]: audit 2024-03-07T03:30:08.343505+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]: dispatch 2024-03-07T03:30:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:08 smithi195 bash[66542]: cluster 2024-03-07T03:30:06.696098+0000 mgr.y (mgr.34101) 485 : cluster 0 pgmap v805: 585 pgs: 12 creating+peering, 37 unknown, 536 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 329 KiB/s rd, 9.5 KiB/s wr, 407 op/s 2024-03-07T03:30:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:08 smithi195 bash[66542]: audit 2024-03-07T03:30:07.978197+0000 mon.a (mon.0) 1186 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:08 smithi195 bash[66542]: audit 2024-03-07T03:30:08.322637+0000 mon.a (mon.0) 1187 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:30:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:08 smithi195 bash[66542]: cluster 2024-03-07T03:30:08.330675+0000 mon.a (mon.0) 1188 : cluster 0 osdmap e703: 8 total, 8 up, 8 in 2024-03-07T03:30:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:08 smithi195 bash[66542]: audit 2024-03-07T03:30:08.337793+0000 mon.c (mon.2) 65 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]: dispatch 2024-03-07T03:30:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:08 smithi195 bash[66542]: audit 2024-03-07T03:30:08.343505+0000 mon.a (mon.0) 1189 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]: dispatch 2024-03-07T03:30:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[79819]: cluster 2024-03-07T03:30:08.698768+0000 mgr.y (mgr.34101) 486 : cluster 0 pgmap v808: 617 pgs: 11 creating+peering, 36 unknown, 570 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.5 KiB/s wr, 214 op/s 2024-03-07T03:30:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[79819]: audit 2024-03-07T03:30:09.325627+0000 mon.a (mon.0) 1190 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]': finished 2024-03-07T03:30:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[79819]: audit 2024-03-07T03:30:09.334875+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]: dispatch 2024-03-07T03:30:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[79819]: cluster 2024-03-07T03:30:09.336613+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2024-03-07T03:30:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[79819]: audit 2024-03-07T03:30:09.340876+0000 mon.a (mon.0) 1192 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]: dispatch 2024-03-07T03:30:10.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[82222]: cluster 2024-03-07T03:30:08.698768+0000 mgr.y (mgr.34101) 486 : cluster 0 pgmap v808: 617 pgs: 11 creating+peering, 36 unknown, 570 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.5 KiB/s wr, 214 op/s 2024-03-07T03:30:10.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[82222]: audit 2024-03-07T03:30:09.325627+0000 mon.a (mon.0) 1190 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]': finished 2024-03-07T03:30:10.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[82222]: audit 2024-03-07T03:30:09.334875+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]: dispatch 2024-03-07T03:30:10.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[82222]: cluster 2024-03-07T03:30:09.336613+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2024-03-07T03:30:10.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:10 smithi148 bash[82222]: audit 2024-03-07T03:30:09.340876+0000 mon.a (mon.0) 1192 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]: dispatch 2024-03-07T03:30:10.814 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:30:10 smithi195 bash[49044]: logger=cleanup t=2024-03-07T03:30:10.556091968Z level=info msg="Completed cleanup jobs" duration=443.177829ms 2024-03-07T03:30:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:10 smithi195 bash[66542]: cluster 2024-03-07T03:30:08.698768+0000 mgr.y (mgr.34101) 486 : cluster 0 pgmap v808: 617 pgs: 11 creating+peering, 36 unknown, 570 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 178 KiB/s rd, 4.5 KiB/s wr, 214 op/s 2024-03-07T03:30:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:10 smithi195 bash[66542]: audit 2024-03-07T03:30:09.325627+0000 mon.a (mon.0) 1190 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31", "force_nonempty":""}]': finished 2024-03-07T03:30:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:10 smithi195 bash[66542]: audit 2024-03-07T03:30:09.334875+0000 mon.c (mon.2) 66 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]: dispatch 2024-03-07T03:30:10.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:10 smithi195 bash[66542]: cluster 2024-03-07T03:30:09.336613+0000 mon.a (mon.0) 1191 : cluster 0 osdmap e704: 8 total, 8 up, 8 in 2024-03-07T03:30:10.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:10 smithi195 bash[66542]: audit 2024-03-07T03:30:09.340876+0000 mon.a (mon.0) 1192 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]: dispatch 2024-03-07T03:30:11.352 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[79819]: cluster 2024-03-07T03:30:10.326213+0000 mon.a (mon.0) 1193 : cluster 3 Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-07T03:30:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[79819]: audit 2024-03-07T03:30:10.328996+0000 mon.a (mon.0) 1194 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]': finished 2024-03-07T03:30:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[79819]: cluster 2024-03-07T03:30:10.338887+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2024-03-07T03:30:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[79819]: audit 2024-03-07T03:30:10.341623+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[79819]: audit 2024-03-07T03:30:10.354024+0000 mon.a (mon.0) 1196 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:11.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[82222]: cluster 2024-03-07T03:30:10.326213+0000 mon.a (mon.0) 1193 : cluster 3 Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-07T03:30:11.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[82222]: audit 2024-03-07T03:30:10.328996+0000 mon.a (mon.0) 1194 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]': finished 2024-03-07T03:30:11.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[82222]: cluster 2024-03-07T03:30:10.338887+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2024-03-07T03:30:11.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[82222]: audit 2024-03-07T03:30:10.341623+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:11.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:11 smithi148 bash[82222]: audit 2024-03-07T03:30:10.354024+0000 mon.a (mon.0) 1196 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:11 smithi195 bash[66542]: cluster 2024-03-07T03:30:10.326213+0000 mon.a (mon.0) 1193 : cluster 3 Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-03-07T03:30:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:11 smithi195 bash[66542]: audit 2024-03-07T03:30:10.328996+0000 mon.a (mon.0) 1194 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi148-68562-31", "mode":"writeback"}]': finished 2024-03-07T03:30:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:11 smithi195 bash[66542]: cluster 2024-03-07T03:30:10.338887+0000 mon.a (mon.0) 1195 : cluster 0 osdmap e705: 8 total, 8 up, 8 in 2024-03-07T03:30:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:11 smithi195 bash[66542]: audit 2024-03-07T03:30:10.341623+0000 mon.c (mon.2) 67 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:11 smithi195 bash[66542]: audit 2024-03-07T03:30:10.354024+0000 mon.a (mon.0) 1196 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:12 smithi195 bash[66542]: cluster 2024-03-07T03:30:10.700999+0000 mgr.y (mgr.34101) 487 : cluster 0 pgmap v811: 617 pgs: 3 creating+peering, 2 unknown, 612 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:30:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:12 smithi195 bash[66542]: audit 2024-03-07T03:30:11.351436+0000 mon.a (mon.0) 1197 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]': finished 2024-03-07T03:30:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:12 smithi195 bash[66542]: cluster 2024-03-07T03:30:11.362639+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2024-03-07T03:30:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:12 smithi195 bash[66542]: cluster 2024-03-07T03:30:12.327928+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2024-03-07T03:30:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:12 smithi195 bash[66542]: audit 2024-03-07T03:30:12.341251+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:12 smithi195 bash[66542]: audit 2024-03-07T03:30:12.345912+0000 mon.a (mon.0) 1201 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:12.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:30:12 smithi195 bash[59134]: ts=2024-03-07T03:30:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[79819]: cluster 2024-03-07T03:30:10.700999+0000 mgr.y (mgr.34101) 487 : cluster 0 pgmap v811: 617 pgs: 3 creating+peering, 2 unknown, 612 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:30:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[79819]: audit 2024-03-07T03:30:11.351436+0000 mon.a (mon.0) 1197 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]': finished 2024-03-07T03:30:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[79819]: cluster 2024-03-07T03:30:11.362639+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2024-03-07T03:30:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[79819]: cluster 2024-03-07T03:30:12.327928+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2024-03-07T03:30:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[79819]: audit 2024-03-07T03:30:12.341251+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[79819]: audit 2024-03-07T03:30:12.345912+0000 mon.a (mon.0) 1201 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:12.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[82222]: cluster 2024-03-07T03:30:10.700999+0000 mgr.y (mgr.34101) 487 : cluster 0 pgmap v811: 617 pgs: 3 creating+peering, 2 unknown, 612 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:30:12.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[82222]: audit 2024-03-07T03:30:11.351436+0000 mon.a (mon.0) 1197 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi148-68562-29", "overlaypool":"test-librbd-smithi148-68562-31"}]': finished 2024-03-07T03:30:12.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[82222]: cluster 2024-03-07T03:30:11.362639+0000 mon.a (mon.0) 1198 : cluster 0 osdmap e706: 8 total, 8 up, 8 in 2024-03-07T03:30:12.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[82222]: cluster 2024-03-07T03:30:12.327928+0000 mon.a (mon.0) 1199 : cluster 0 osdmap e707: 8 total, 8 up, 8 in 2024-03-07T03:30:12.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[82222]: audit 2024-03-07T03:30:12.341251+0000 mon.a (mon.0) 1200 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:12.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:12 smithi148 bash[82222]: audit 2024-03-07T03:30:12.345912+0000 mon.a (mon.0) 1201 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:14.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:14 smithi148 bash[79819]: cluster 2024-03-07T03:30:12.702074+0000 mgr.y (mgr.34101) 488 : cluster 0 pgmap v814: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:30:14.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:14 smithi148 bash[79819]: cluster 2024-03-07T03:30:13.370905+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2024-03-07T03:30:14.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:14 smithi148 bash[82222]: cluster 2024-03-07T03:30:12.702074+0000 mgr.y (mgr.34101) 488 : cluster 0 pgmap v814: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:30:14.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:14 smithi148 bash[82222]: cluster 2024-03-07T03:30:13.370905+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2024-03-07T03:30:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:14 smithi195 bash[66542]: cluster 2024-03-07T03:30:12.702074+0000 mgr.y (mgr.34101) 488 : cluster 0 pgmap v814: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:30:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:14 smithi195 bash[66542]: cluster 2024-03-07T03:30:13.370905+0000 mon.a (mon.0) 1202 : cluster 0 osdmap e708: 8 total, 8 up, 8 in 2024-03-07T03:30:15.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[79819]: cluster 2024-03-07T03:30:14.375304+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2024-03-07T03:30:15.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[79819]: audit 2024-03-07T03:30:14.820593+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-03-07T03:30:15.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[79819]: audit 2024-03-07T03:30:14.821295+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]: dispatch 2024-03-07T03:30:15.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[79819]: audit 2024-03-07T03:30:14.821954+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.13", "id": [3, 5]}]: dispatch 2024-03-07T03:30:15.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[79819]: audit 2024-03-07T03:30:14.822618+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]: dispatch 2024-03-07T03:30:15.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[79819]: audit 2024-03-07T03:30:14.823255+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]: dispatch 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[79819]: audit 2024-03-07T03:30:14.823928+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[82222]: cluster 2024-03-07T03:30:14.375304+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[82222]: audit 2024-03-07T03:30:14.820593+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[82222]: audit 2024-03-07T03:30:14.821295+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]: dispatch 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[82222]: audit 2024-03-07T03:30:14.821954+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.13", "id": [3, 5]}]: dispatch 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[82222]: audit 2024-03-07T03:30:14.822618+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]: dispatch 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[82222]: audit 2024-03-07T03:30:14.823255+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]: dispatch 2024-03-07T03:30:15.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:15 smithi148 bash[82222]: audit 2024-03-07T03:30:14.823928+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2024-03-07T03:30:15.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:15 smithi195 bash[66542]: cluster 2024-03-07T03:30:14.375304+0000 mon.a (mon.0) 1203 : cluster 0 osdmap e709: 8 total, 8 up, 8 in 2024-03-07T03:30:15.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:15 smithi195 bash[66542]: audit 2024-03-07T03:30:14.820593+0000 mon.a (mon.0) 1204 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]: dispatch 2024-03-07T03:30:15.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:15 smithi195 bash[66542]: audit 2024-03-07T03:30:14.821295+0000 mon.a (mon.0) 1205 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]: dispatch 2024-03-07T03:30:15.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:15 smithi195 bash[66542]: audit 2024-03-07T03:30:14.821954+0000 mon.a (mon.0) 1206 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.13", "id": [3, 5]}]: dispatch 2024-03-07T03:30:15.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:15 smithi195 bash[66542]: audit 2024-03-07T03:30:14.822618+0000 mon.a (mon.0) 1207 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]: dispatch 2024-03-07T03:30:15.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:15 smithi195 bash[66542]: audit 2024-03-07T03:30:14.823255+0000 mon.a (mon.0) 1208 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]: dispatch 2024-03-07T03:30:15.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:15 smithi195 bash[66542]: audit 2024-03-07T03:30:14.823928+0000 mon.a (mon.0) 1209 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]: dispatch 2024-03-07T03:30:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: cluster 2024-03-07T03:30:14.703166+0000 mgr.y (mgr.34101) 489 : cluster 0 pgmap v817: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 2 op/s 2024-03-07T03:30:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: audit 2024-03-07T03:30:15.390603+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]': finished 2024-03-07T03:30:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: audit 2024-03-07T03:30:15.390753+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]': finished 2024-03-07T03:30:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: audit 2024-03-07T03:30:15.390921+0000 mon.a (mon.0) 1212 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.13", "id": [3, 5]}]': finished 2024-03-07T03:30:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: audit 2024-03-07T03:30:15.391156+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]': finished 2024-03-07T03:30:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: audit 2024-03-07T03:30:15.391365+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]': finished 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: audit 2024-03-07T03:30:15.391472+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]': finished 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[79819]: cluster 2024-03-07T03:30:15.401836+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: cluster 2024-03-07T03:30:14.703166+0000 mgr.y (mgr.34101) 489 : cluster 0 pgmap v817: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 2 op/s 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: audit 2024-03-07T03:30:15.390603+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]': finished 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: audit 2024-03-07T03:30:15.390753+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]': finished 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: audit 2024-03-07T03:30:15.390921+0000 mon.a (mon.0) 1212 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.13", "id": [3, 5]}]': finished 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: audit 2024-03-07T03:30:15.391156+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]': finished 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: audit 2024-03-07T03:30:15.391365+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]': finished 2024-03-07T03:30:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: audit 2024-03-07T03:30:15.391472+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]': finished 2024-03-07T03:30:16.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:16 smithi148 bash[82222]: cluster 2024-03-07T03:30:15.401836+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2024-03-07T03:30:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: cluster 2024-03-07T03:30:14.703166+0000 mgr.y (mgr.34101) 489 : cluster 0 pgmap v817: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 767 B/s wr, 2 op/s 2024-03-07T03:30:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: audit 2024-03-07T03:30:15.390603+0000 mon.a (mon.0) 1210 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.2", "id": [2, 4]}]': finished 2024-03-07T03:30:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: audit 2024-03-07T03:30:15.390753+0000 mon.a (mon.0) 1211 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]': finished 2024-03-07T03:30:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: audit 2024-03-07T03:30:15.390921+0000 mon.a (mon.0) 1212 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.13", "id": [3, 5]}]': finished 2024-03-07T03:30:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: audit 2024-03-07T03:30:15.391156+0000 mon.a (mon.0) 1213 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.15", "id": [2, 5]}]': finished 2024-03-07T03:30:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: audit 2024-03-07T03:30:15.391365+0000 mon.a (mon.0) 1214 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]': finished 2024-03-07T03:30:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: audit 2024-03-07T03:30:15.391472+0000 mon.a (mon.0) 1215 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.4", "id": [4, 2]}]': finished 2024-03-07T03:30:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:16 smithi195 bash[66542]: cluster 2024-03-07T03:30:15.401836+0000 mon.a (mon.0) 1216 : cluster 0 osdmap e710: 8 total, 8 up, 8 in 2024-03-07T03:30:17.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:17 smithi148 bash[79819]: cluster 2024-03-07T03:30:16.400325+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2024-03-07T03:30:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:17 smithi148 bash[82222]: cluster 2024-03-07T03:30:16.400325+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2024-03-07T03:30:17.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:30:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:30:17] "GET /metrics HTTP/1.1" 200 75116 "" "Prometheus/2.43.0" 2024-03-07T03:30:17.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:17 smithi195 bash[66542]: cluster 2024-03-07T03:30:16.400325+0000 mon.a (mon.0) 1217 : cluster 0 osdmap e711: 8 total, 8 up, 8 in 2024-03-07T03:30:17.831 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.831 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.831 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.831 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.831 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.831 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 1 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout:child: image118 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118/acb81e16b597 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.832 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 2 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:child: image118 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118/acb81e16b597 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.833 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.834 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 3 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:child: image118 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:child: image120 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118/acb81e16b597 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.835 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120/acb8b346873a 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.836 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 3 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:child: image118 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:child: image120 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118/acb81e16b597 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120/acb8b346873a 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.837 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:17.838 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 4 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 4 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:child: image118 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:child: image120 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:child: image121 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118/acb81e16b597 2024-03-07T03:30:17.839 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120/acb8b346873a 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121/acb8beeb9416 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.840 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.841 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:17.842 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:17.842 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:18.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:18 smithi148 bash[79819]: cluster 2024-03-07T03:30:16.704261+0000 mgr.y (mgr.34101) 490 : cluster 0 pgmap v820: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-03-07T03:30:18.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:18 smithi148 bash[79819]: audit 2024-03-07T03:30:18.038653+0000 mon.a (mon.0) 1218 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:18 smithi148 bash[82222]: cluster 2024-03-07T03:30:16.704261+0000 mgr.y (mgr.34101) 490 : cluster 0 pgmap v820: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-03-07T03:30:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:18 smithi148 bash[82222]: audit 2024-03-07T03:30:18.038653+0000 mon.a (mon.0) 1218 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:18 smithi195 bash[66542]: cluster 2024-03-07T03:30:16.704261+0000 mgr.y (mgr.34101) 490 : cluster 0 pgmap v820: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.5 KiB/s wr, 6 op/s 2024-03-07T03:30:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:18 smithi195 bash[66542]: audit 2024-03-07T03:30:18.038653+0000 mon.a (mon.0) 1218 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:20.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[79819]: cluster 2024-03-07T03:30:18.707422+0000 mgr.y (mgr.34101) 491 : cluster 0 pgmap v821: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 248 KiB/s rd, 6.2 KiB/s wr, 299 op/s 2024-03-07T03:30:20.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[79819]: cluster 2024-03-07T03:30:19.442407+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2024-03-07T03:30:20.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[79819]: audit 2024-03-07T03:30:19.705210+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]: dispatch 2024-03-07T03:30:20.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[79819]: audit 2024-03-07T03:30:19.705907+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]: dispatch 2024-03-07T03:30:20.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[82222]: cluster 2024-03-07T03:30:18.707422+0000 mgr.y (mgr.34101) 491 : cluster 0 pgmap v821: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 248 KiB/s rd, 6.2 KiB/s wr, 299 op/s 2024-03-07T03:30:20.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[82222]: cluster 2024-03-07T03:30:19.442407+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2024-03-07T03:30:20.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[82222]: audit 2024-03-07T03:30:19.705210+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]: dispatch 2024-03-07T03:30:20.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:20 smithi148 bash[82222]: audit 2024-03-07T03:30:19.705907+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]: dispatch 2024-03-07T03:30:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:20 smithi195 bash[66542]: cluster 2024-03-07T03:30:18.707422+0000 mgr.y (mgr.34101) 491 : cluster 0 pgmap v821: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 248 KiB/s rd, 6.2 KiB/s wr, 299 op/s 2024-03-07T03:30:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:20 smithi195 bash[66542]: cluster 2024-03-07T03:30:19.442407+0000 mon.a (mon.0) 1219 : cluster 0 osdmap e712: 8 total, 8 up, 8 in 2024-03-07T03:30:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:20 smithi195 bash[66542]: audit 2024-03-07T03:30:19.705210+0000 mon.c (mon.2) 68 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]: dispatch 2024-03-07T03:30:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:20 smithi195 bash[66542]: audit 2024-03-07T03:30:19.705907+0000 mon.a (mon.0) 1220 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]: dispatch 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image118 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 4 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 4 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:21.437 INFO:tasks.workunit.client.0.smithi148.stdout:child: image118 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:child: image120 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:child: image121 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image118/acb81e16b597 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120/acb8b346873a 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121/acb8beeb9416 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.438 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image120 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.439 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 3 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:child: image120 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:child: image121 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image120/acb8b346873a 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121/acb8beeb9416 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.440 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 2 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 2 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:21.441 INFO:tasks.workunit.client.0.smithi148.stdout:child: image121 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-30/image121/acb8beeb9416 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:trying to find test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:checking test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 1 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 1 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:child: image119 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-29/image119/acb8d7bbb09f 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 0 2024-03-07T03:30:21.442 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:30:21.443 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ListChildrenTiered (18826 ms) 2024-03-07T03:30:21.443 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.LockingPP 2024-03-07T03:30:21.443 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:21.612 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.LockingPP (175 ms) 2024-03-07T03:30:21.612 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.FlushAio 2024-03-07T03:30:21.612 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:21.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[79819]: audit 2024-03-07T03:30:20.439953+0000 mon.a (mon.0) 1221 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]': finished 2024-03-07T03:30:21.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[79819]: cluster 2024-03-07T03:30:20.446430+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2024-03-07T03:30:21.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[79819]: audit 2024-03-07T03:30:20.447744+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:21.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[79819]: audit 2024-03-07T03:30:20.451430+0000 mon.a (mon.0) 1223 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:21.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[79819]: cluster 2024-03-07T03:30:21.440495+0000 mon.a (mon.0) 1224 : cluster 1 Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[79819]: audit 2024-03-07T03:30:21.442922+0000 mon.a (mon.0) 1225 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]': finished 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[79819]: cluster 2024-03-07T03:30:21.449496+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[82222]: audit 2024-03-07T03:30:20.439953+0000 mon.a (mon.0) 1221 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]': finished 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[82222]: cluster 2024-03-07T03:30:20.446430+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[82222]: audit 2024-03-07T03:30:20.447744+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[82222]: audit 2024-03-07T03:30:20.451430+0000 mon.a (mon.0) 1223 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[82222]: cluster 2024-03-07T03:30:21.440495+0000 mon.a (mon.0) 1224 : cluster 1 Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[82222]: audit 2024-03-07T03:30:21.442922+0000 mon.a (mon.0) 1225 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]': finished 2024-03-07T03:30:21.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:21 smithi148 bash[82222]: cluster 2024-03-07T03:30:21.449496+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2024-03-07T03:30:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:21 smithi195 bash[66542]: audit 2024-03-07T03:30:20.439953+0000 mon.a (mon.0) 1221 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi148-68562-29"}]': finished 2024-03-07T03:30:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:21 smithi195 bash[66542]: cluster 2024-03-07T03:30:20.446430+0000 mon.a (mon.0) 1222 : cluster 0 osdmap e713: 8 total, 8 up, 8 in 2024-03-07T03:30:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:21 smithi195 bash[66542]: audit 2024-03-07T03:30:20.447744+0000 mon.c (mon.2) 69 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:21 smithi195 bash[66542]: audit 2024-03-07T03:30:20.451430+0000 mon.a (mon.0) 1223 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]: dispatch 2024-03-07T03:30:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:21 smithi195 bash[66542]: cluster 2024-03-07T03:30:21.440495+0000 mon.a (mon.0) 1224 : cluster 1 Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-03-07T03:30:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:21 smithi195 bash[66542]: audit 2024-03-07T03:30:21.442922+0000 mon.a (mon.0) 1225 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi148-68562-29", "tierpool":"test-librbd-smithi148-68562-31"}]': finished 2024-03-07T03:30:21.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:21 smithi195 bash[66542]: cluster 2024-03-07T03:30:21.449496+0000 mon.a (mon.0) 1226 : cluster 0 osdmap e714: 8 total, 8 up, 8 in 2024-03-07T03:30:22.038 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.FlushAio (426 ms) 2024-03-07T03:30:22.038 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-03-07T03:30:22.038 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:22.412 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.FlushAioPP (374 ms) 2024-03-07T03:30:22.412 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-03-07T03:30:22.413 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:22.452 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (40 ms) 2024-03-07T03:30:22.452 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-03-07T03:30:22.452 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:22.482 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (31 ms) 2024-03-07T03:30:22.483 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-03-07T03:30:22.483 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:22.507 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ZeroLengthRead (24 ms) 2024-03-07T03:30:22.507 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-03-07T03:30:22.507 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:22.559 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.LargeCacheRead (52 ms) 2024-03-07T03:30:22.559 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-03-07T03:30:22.559 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:22 smithi195 bash[66542]: cluster 2024-03-07T03:30:20.708603+0000 mgr.y (mgr.34101) 492 : cluster 0 pgmap v824: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 395 KiB/s rd, 8.3 KiB/s wr, 474 op/s 2024-03-07T03:30:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:22 smithi195 bash[66542]: cluster 2024-03-07T03:30:22.329243+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2024-03-07T03:30:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:30:22 smithi195 bash[59134]: ts=2024-03-07T03:30:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:22.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:22 smithi148 bash[79819]: cluster 2024-03-07T03:30:20.708603+0000 mgr.y (mgr.34101) 492 : cluster 0 pgmap v824: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 395 KiB/s rd, 8.3 KiB/s wr, 474 op/s 2024-03-07T03:30:22.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:22 smithi148 bash[79819]: cluster 2024-03-07T03:30:22.329243+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2024-03-07T03:30:22.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:22 smithi148 bash[82222]: cluster 2024-03-07T03:30:20.708603+0000 mgr.y (mgr.34101) 492 : cluster 0 pgmap v824: 617 pgs: 2 peering, 615 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 395 KiB/s rd, 8.3 KiB/s wr, 474 op/s 2024-03-07T03:30:22.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:22 smithi148 bash[82222]: cluster 2024-03-07T03:30:22.329243+0000 mon.a (mon.0) 1227 : cluster 0 osdmap e715: 8 total, 8 up, 8 in 2024-03-07T03:30:22.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestPendingAio (271 ms) 2024-03-07T03:30:22.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-03-07T03:30:22.830 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:23.725 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:24 smithi148 bash[79819]: cluster 2024-03-07T03:30:22.709330+0000 mgr.y (mgr.34101) 493 : cluster 0 pgmap v827: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 209 KiB/s rd, 4.0 KiB/s wr, 253 op/s 2024-03-07T03:30:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:24 smithi148 bash[79819]: cluster 2024-03-07T03:30:23.485356+0000 mon.a (mon.0) 1228 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2024-03-07T03:30:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:24 smithi148 bash[82222]: cluster 2024-03-07T03:30:22.709330+0000 mgr.y (mgr.34101) 493 : cluster 0 pgmap v827: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 209 KiB/s rd, 4.0 KiB/s wr, 253 op/s 2024-03-07T03:30:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:24 smithi148 bash[82222]: cluster 2024-03-07T03:30:23.485356+0000 mon.a (mon.0) 1228 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2024-03-07T03:30:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:24 smithi195 bash[66542]: cluster 2024-03-07T03:30:22.709330+0000 mgr.y (mgr.34101) 493 : cluster 0 pgmap v827: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 209 KiB/s rd, 4.0 KiB/s wr, 253 op/s 2024-03-07T03:30:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:24 smithi195 bash[66542]: cluster 2024-03-07T03:30:23.485356+0000 mon.a (mon.0) 1228 : cluster 0 osdmap e716: 8 total, 8 up, 8 in 2024-03-07T03:30:25.600 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2770 ms) 2024-03-07T03:30:25.600 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-03-07T03:30:25.600 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:25.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:25 smithi148 bash[79819]: cluster 2024-03-07T03:30:24.496373+0000 mon.a (mon.0) 1229 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2024-03-07T03:30:25.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:25 smithi148 bash[82222]: cluster 2024-03-07T03:30:24.496373+0000 mon.a (mon.0) 1229 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2024-03-07T03:30:25.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:25 smithi195 bash[66542]: cluster 2024-03-07T03:30:24.496373+0000 mon.a (mon.0) 1229 : cluster 0 osdmap e717: 8 total, 8 up, 8 in 2024-03-07T03:30:26.710 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:26.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:26 smithi148 bash[79819]: cluster 2024-03-07T03:30:24.710491+0000 mgr.y (mgr.34101) 494 : cluster 0 pgmap v830: 617 pgs: 617 active+clean; 303 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 278 KiB/s rd, 208 KiB/s wr, 547 op/s 2024-03-07T03:30:26.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:26 smithi148 bash[79819]: cluster 2024-03-07T03:30:25.506867+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2024-03-07T03:30:26.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:26 smithi148 bash[82222]: cluster 2024-03-07T03:30:24.710491+0000 mgr.y (mgr.34101) 494 : cluster 0 pgmap v830: 617 pgs: 617 active+clean; 303 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 278 KiB/s rd, 208 KiB/s wr, 547 op/s 2024-03-07T03:30:26.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:26 smithi148 bash[82222]: cluster 2024-03-07T03:30:25.506867+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2024-03-07T03:30:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:26 smithi195 bash[66542]: cluster 2024-03-07T03:30:24.710491+0000 mgr.y (mgr.34101) 494 : cluster 0 pgmap v830: 617 pgs: 617 active+clean; 303 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 278 KiB/s rd, 208 KiB/s wr, 547 op/s 2024-03-07T03:30:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:26 smithi195 bash[66542]: cluster 2024-03-07T03:30:25.506867+0000 mon.a (mon.0) 1230 : cluster 0 osdmap e718: 8 total, 8 up, 8 in 2024-03-07T03:30:27.505 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:30:27] "GET /metrics HTTP/1.1" 200 75116 "" "Prometheus/2.43.0" 2024-03-07T03:30:27.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[79819]: cluster 2024-03-07T03:30:26.513778+0000 mon.a (mon.0) 1231 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2024-03-07T03:30:27.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[79819]: audit 2024-03-07T03:30:27.331047+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:27.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[79819]: audit 2024-03-07T03:30:27.334499+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:27.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[79819]: cluster 2024-03-07T03:30:27.343753+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2024-03-07T03:30:27.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[82222]: cluster 2024-03-07T03:30:26.513778+0000 mon.a (mon.0) 1231 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2024-03-07T03:30:27.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[82222]: audit 2024-03-07T03:30:27.331047+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:27.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[82222]: audit 2024-03-07T03:30:27.334499+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:27.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:27 smithi148 bash[82222]: cluster 2024-03-07T03:30:27.343753+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2024-03-07T03:30:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:27 smithi195 bash[66542]: cluster 2024-03-07T03:30:26.513778+0000 mon.a (mon.0) 1231 : cluster 0 osdmap e719: 8 total, 8 up, 8 in 2024-03-07T03:30:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:27 smithi195 bash[66542]: audit 2024-03-07T03:30:27.331047+0000 mon.a (mon.0) 1232 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:27 smithi195 bash[66542]: audit 2024-03-07T03:30:27.334499+0000 mon.a (mon.0) 1233 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:27 smithi195 bash[66542]: cluster 2024-03-07T03:30:27.343753+0000 mon.a (mon.0) 1234 : cluster 0 osdmap e720: 8 total, 8 up, 8 in 2024-03-07T03:30:28.623 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (3023 ms) 2024-03-07T03:30:28.624 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.Flatten 2024-03-07T03:30:28.624 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:28 smithi148 bash[82222]: cluster 2024-03-07T03:30:26.711680+0000 mgr.y (mgr.34101) 495 : cluster 0 pgmap v833: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 846 KiB/s rd, 730 KiB/s wr, 632 op/s 2024-03-07T03:30:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:28 smithi148 bash[82222]: audit 2024-03-07T03:30:28.123852+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:28.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:28 smithi148 bash[79819]: cluster 2024-03-07T03:30:26.711680+0000 mgr.y (mgr.34101) 495 : cluster 0 pgmap v833: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 846 KiB/s rd, 730 KiB/s wr, 632 op/s 2024-03-07T03:30:28.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:28 smithi148 bash[79819]: audit 2024-03-07T03:30:28.123852+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:28 smithi195 bash[66542]: cluster 2024-03-07T03:30:26.711680+0000 mgr.y (mgr.34101) 495 : cluster 0 pgmap v833: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 846 KiB/s rd, 730 KiB/s wr, 632 op/s 2024-03-07T03:30:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:28 smithi195 bash[66542]: audit 2024-03-07T03:30:28.123852+0000 mon.a (mon.0) 1235 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:29.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:29 smithi148 bash[79819]: cluster 2024-03-07T03:30:28.526898+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2024-03-07T03:30:29.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:29 smithi148 bash[82222]: cluster 2024-03-07T03:30:28.526898+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2024-03-07T03:30:29.804 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.Flatten (1181 ms) 2024-03-07T03:30:29.804 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.Sparsify 2024-03-07T03:30:29.805 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:29.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:29 smithi195 bash[66542]: cluster 2024-03-07T03:30:28.526898+0000 mon.a (mon.0) 1236 : cluster 0 osdmap e721: 8 total, 8 up, 8 in 2024-03-07T03:30:29.889 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 32768 2024-03-07T03:30:29.889 INFO:tasks.workunit.client.0.smithi148.stdout:read: 32768 2024-03-07T03:30:29.890 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.Sparsify (85 ms) 2024-03-07T03:30:29.890 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-03-07T03:30:29.890 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:29.977 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SparsifyPP (88 ms) 2024-03-07T03:30:29.977 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-03-07T03:30:29.977 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:30.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:30 smithi148 bash[79819]: cluster 2024-03-07T03:30:28.712842+0000 mgr.y (mgr.34101) 496 : cluster 0 pgmap v836: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 648 KiB/s rd, 531 KiB/s wr, 196 op/s 2024-03-07T03:30:30.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:30 smithi148 bash[79819]: cluster 2024-03-07T03:30:29.552747+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2024-03-07T03:30:30.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:30 smithi148 bash[82222]: cluster 2024-03-07T03:30:28.712842+0000 mgr.y (mgr.34101) 496 : cluster 0 pgmap v836: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 648 KiB/s rd, 531 KiB/s wr, 196 op/s 2024-03-07T03:30:30.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:30 smithi148 bash[82222]: cluster 2024-03-07T03:30:29.552747+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2024-03-07T03:30:30.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:30 smithi195 bash[66542]: cluster 2024-03-07T03:30:28.712842+0000 mgr.y (mgr.34101) 496 : cluster 0 pgmap v836: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 648 KiB/s rd, 531 KiB/s wr, 196 op/s 2024-03-07T03:30:30.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:30 smithi195 bash[66542]: cluster 2024-03-07T03:30:29.552747+0000 mon.a (mon.0) 1237 : cluster 0 osdmap e722: 8 total, 8 up, 8 in 2024-03-07T03:30:31.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:31 smithi195 bash[66542]: cluster 2024-03-07T03:30:30.558897+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2024-03-07T03:30:32.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:31 smithi148 bash[79819]: cluster 2024-03-07T03:30:30.558897+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2024-03-07T03:30:32.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:31 smithi148 bash[82222]: cluster 2024-03-07T03:30:30.558897+0000 mon.a (mon.0) 1238 : cluster 0 osdmap e723: 8 total, 8 up, 8 in 2024-03-07T03:30:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:30:32 smithi195 bash[59134]: ts=2024-03-07T03:30:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:33.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:32 smithi148 bash[79819]: cluster 2024-03-07T03:30:30.713959+0000 mgr.y (mgr.34101) 497 : cluster 0 pgmap v839: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 143 KiB/s rd, 19 KiB/s wr, 196 op/s 2024-03-07T03:30:33.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:32 smithi148 bash[79819]: cluster 2024-03-07T03:30:31.576741+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2024-03-07T03:30:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:32 smithi148 bash[82222]: cluster 2024-03-07T03:30:30.713959+0000 mgr.y (mgr.34101) 497 : cluster 0 pgmap v839: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 143 KiB/s rd, 19 KiB/s wr, 196 op/s 2024-03-07T03:30:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:32 smithi148 bash[82222]: cluster 2024-03-07T03:30:31.576741+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2024-03-07T03:30:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:32 smithi195 bash[66542]: cluster 2024-03-07T03:30:30.713959+0000 mgr.y (mgr.34101) 497 : cluster 0 pgmap v839: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 143 KiB/s rd, 19 KiB/s wr, 196 op/s 2024-03-07T03:30:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:32 smithi195 bash[66542]: cluster 2024-03-07T03:30:31.576741+0000 mon.a (mon.0) 1239 : cluster 0 osdmap e724: 8 total, 8 up, 8 in 2024-03-07T03:30:34.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:33 smithi148 bash[79819]: cluster 2024-03-07T03:30:32.575179+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2024-03-07T03:30:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:33 smithi148 bash[82222]: cluster 2024-03-07T03:30:32.575179+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2024-03-07T03:30:34.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:33 smithi195 bash[66542]: cluster 2024-03-07T03:30:32.575179+0000 mon.a (mon.0) 1240 : cluster 0 osdmap e725: 8 total, 8 up, 8 in 2024-03-07T03:30:34.664 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapshotLimit (4687 ms) 2024-03-07T03:30:34.665 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-03-07T03:30:34.665 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:35.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:34 smithi148 bash[79819]: cluster 2024-03-07T03:30:32.715115+0000 mgr.y (mgr.34101) 498 : cluster 0 pgmap v842: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 125 KiB/s rd, 15 KiB/s wr, 168 op/s 2024-03-07T03:30:35.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:34 smithi148 bash[79819]: cluster 2024-03-07T03:30:33.603602+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2024-03-07T03:30:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:34 smithi148 bash[82222]: cluster 2024-03-07T03:30:32.715115+0000 mgr.y (mgr.34101) 498 : cluster 0 pgmap v842: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 125 KiB/s rd, 15 KiB/s wr, 168 op/s 2024-03-07T03:30:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:34 smithi148 bash[82222]: cluster 2024-03-07T03:30:33.603602+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2024-03-07T03:30:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:34 smithi195 bash[66542]: cluster 2024-03-07T03:30:32.715115+0000 mgr.y (mgr.34101) 498 : cluster 0 pgmap v842: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 125 KiB/s rd, 15 KiB/s wr, 168 op/s 2024-03-07T03:30:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:34 smithi195 bash[66542]: cluster 2024-03-07T03:30:33.603602+0000 mon.a (mon.0) 1241 : cluster 0 osdmap e726: 8 total, 8 up, 8 in 2024-03-07T03:30:36.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:35 smithi148 bash[79819]: cluster 2024-03-07T03:30:34.608113+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2024-03-07T03:30:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:35 smithi148 bash[82222]: cluster 2024-03-07T03:30:34.608113+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2024-03-07T03:30:36.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:35 smithi195 bash[66542]: cluster 2024-03-07T03:30:34.608113+0000 mon.a (mon.0) 1242 : cluster 0 osdmap e727: 8 total, 8 up, 8 in 2024-03-07T03:30:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:36 smithi148 bash[79819]: cluster 2024-03-07T03:30:34.716030+0000 mgr.y (mgr.34101) 499 : cluster 0 pgmap v845: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 30 KiB/s wr, 183 op/s 2024-03-07T03:30:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:36 smithi148 bash[79819]: cluster 2024-03-07T03:30:35.623699+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2024-03-07T03:30:37.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:36 smithi148 bash[82222]: cluster 2024-03-07T03:30:34.716030+0000 mgr.y (mgr.34101) 499 : cluster 0 pgmap v845: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 30 KiB/s wr, 183 op/s 2024-03-07T03:30:37.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:36 smithi148 bash[82222]: cluster 2024-03-07T03:30:35.623699+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2024-03-07T03:30:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:36 smithi195 bash[66542]: cluster 2024-03-07T03:30:34.716030+0000 mgr.y (mgr.34101) 499 : cluster 0 pgmap v845: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 154 KiB/s rd, 30 KiB/s wr, 183 op/s 2024-03-07T03:30:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:36 smithi195 bash[66542]: cluster 2024-03-07T03:30:35.623699+0000 mon.a (mon.0) 1243 : cluster 0 osdmap e728: 8 total, 8 up, 8 in 2024-03-07T03:30:37.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:30:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:30:37] "GET /metrics HTTP/1.1" 200 75186 "" "Prometheus/2.43.0" 2024-03-07T03:30:38.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:37 smithi148 bash[79819]: cluster 2024-03-07T03:30:36.630727+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2024-03-07T03:30:38.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:37 smithi148 bash[82222]: cluster 2024-03-07T03:30:36.630727+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2024-03-07T03:30:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:37 smithi195 bash[66542]: cluster 2024-03-07T03:30:36.630727+0000 mon.a (mon.0) 1244 : cluster 0 osdmap e729: 8 total, 8 up, 8 in 2024-03-07T03:30:39.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:38 smithi148 bash[79819]: cluster 2024-03-07T03:30:36.717403+0000 mgr.y (mgr.34101) 500 : cluster 0 pgmap v848: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 28 KiB/s wr, 138 op/s 2024-03-07T03:30:39.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:38 smithi148 bash[79819]: cluster 2024-03-07T03:30:37.642041+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2024-03-07T03:30:39.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:38 smithi148 bash[82222]: cluster 2024-03-07T03:30:36.717403+0000 mgr.y (mgr.34101) 500 : cluster 0 pgmap v848: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 28 KiB/s wr, 138 op/s 2024-03-07T03:30:39.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:38 smithi148 bash[82222]: cluster 2024-03-07T03:30:37.642041+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2024-03-07T03:30:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:38 smithi195 bash[66542]: cluster 2024-03-07T03:30:36.717403+0000 mgr.y (mgr.34101) 500 : cluster 0 pgmap v848: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 28 KiB/s wr, 138 op/s 2024-03-07T03:30:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:38 smithi195 bash[66542]: cluster 2024-03-07T03:30:37.642041+0000 mon.a (mon.0) 1245 : cluster 0 osdmap e730: 8 total, 8 up, 8 in 2024-03-07T03:30:39.721 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5057 ms) 2024-03-07T03:30:39.721 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-03-07T03:30:39.721 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:39.868 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:30:39.868 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (147 ms) 2024-03-07T03:30:39.868 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-03-07T03:30:39.868 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:40.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:39 smithi148 bash[79819]: cluster 2024-03-07T03:30:38.642064+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2024-03-07T03:30:40.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:39 smithi148 bash[82222]: cluster 2024-03-07T03:30:38.642064+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2024-03-07T03:30:40.047 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (179 ms) 2024-03-07T03:30:40.047 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-03-07T03:30:40.047 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:40.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:39 smithi195 bash[66542]: cluster 2024-03-07T03:30:38.642064+0000 mon.a (mon.0) 1246 : cluster 0 osdmap e731: 8 total, 8 up, 8 in 2024-03-07T03:30:40.755 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (708 ms) 2024-03-07T03:30:40.755 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-03-07T03:30:40.755 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:41.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:40 smithi148 bash[79819]: cluster 2024-03-07T03:30:38.718666+0000 mgr.y (mgr.34101) 501 : cluster 0 pgmap v851: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 3.7 KiB/s wr, 44 op/s 2024-03-07T03:30:41.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:40 smithi148 bash[79819]: cluster 2024-03-07T03:30:39.664577+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2024-03-07T03:30:41.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:40 smithi148 bash[82222]: cluster 2024-03-07T03:30:38.718666+0000 mgr.y (mgr.34101) 501 : cluster 0 pgmap v851: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 3.7 KiB/s wr, 44 op/s 2024-03-07T03:30:41.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:40 smithi148 bash[82222]: cluster 2024-03-07T03:30:39.664577+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2024-03-07T03:30:41.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:40 smithi195 bash[66542]: cluster 2024-03-07T03:30:38.718666+0000 mgr.y (mgr.34101) 501 : cluster 0 pgmap v851: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 3.7 KiB/s wr, 44 op/s 2024-03-07T03:30:41.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:40 smithi195 bash[66542]: cluster 2024-03-07T03:30:39.664577+0000 mon.a (mon.0) 1247 : cluster 0 osdmap e732: 8 total, 8 up, 8 in 2024-03-07T03:30:42.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:41 smithi148 bash[79819]: cluster 2024-03-07T03:30:40.673299+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2024-03-07T03:30:42.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:41 smithi148 bash[82222]: cluster 2024-03-07T03:30:40.673299+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2024-03-07T03:30:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:41 smithi195 bash[66542]: cluster 2024-03-07T03:30:40.673299+0000 mon.a (mon.0) 1248 : cluster 0 osdmap e733: 8 total, 8 up, 8 in 2024-03-07T03:30:42.386 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (1631 ms) 2024-03-07T03:30:42.386 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-03-07T03:30:42.386 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:42.467 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (81 ms) 2024-03-07T03:30:42.467 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-03-07T03:30:42.467 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_librbd.cc:9061: Skipped 2024-03-07T03:30:42.468 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:30:42.468 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-03-07T03:30:42.468 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-03-07T03:30:42.468 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[59134]: ts=2024-03-07T03:30:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[79819]: cluster 2024-03-07T03:30:40.720140+0000 mgr.y (mgr.34101) 502 : cluster 0 pgmap v854: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 KiB/s wr, 33 op/s 2024-03-07T03:30:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[79819]: cluster 2024-03-07T03:30:41.687702+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2024-03-07T03:30:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[79819]: audit 2024-03-07T03:30:41.694375+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:30:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[79819]: audit 2024-03-07T03:30:42.223103+0000 mon.a (mon.0) 1251 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[79819]: audit 2024-03-07T03:30:42.236360+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[79819]: audit 2024-03-07T03:30:42.323575+0000 mon.a (mon.0) 1253 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:43.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[79819]: cluster 2024-03-07T03:30:42.338481+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2024-03-07T03:30:43.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[82222]: cluster 2024-03-07T03:30:40.720140+0000 mgr.y (mgr.34101) 502 : cluster 0 pgmap v854: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 KiB/s wr, 33 op/s 2024-03-07T03:30:43.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[82222]: cluster 2024-03-07T03:30:41.687702+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2024-03-07T03:30:43.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[82222]: audit 2024-03-07T03:30:41.694375+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:30:43.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[82222]: audit 2024-03-07T03:30:42.223103+0000 mon.a (mon.0) 1251 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:43.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[82222]: audit 2024-03-07T03:30:42.236360+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:43.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[82222]: audit 2024-03-07T03:30:42.323575+0000 mon.a (mon.0) 1253 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:43.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:42 smithi148 bash[82222]: cluster 2024-03-07T03:30:42.338481+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2024-03-07T03:30:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[66542]: cluster 2024-03-07T03:30:40.720140+0000 mgr.y (mgr.34101) 502 : cluster 0 pgmap v854: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.7 KiB/s wr, 33 op/s 2024-03-07T03:30:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[66542]: cluster 2024-03-07T03:30:41.687702+0000 mon.a (mon.0) 1249 : cluster 0 osdmap e734: 8 total, 8 up, 8 in 2024-03-07T03:30:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[66542]: audit 2024-03-07T03:30:41.694375+0000 mon.a (mon.0) 1250 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:30:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[66542]: audit 2024-03-07T03:30:42.223103+0000 mon.a (mon.0) 1251 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[66542]: audit 2024-03-07T03:30:42.236360+0000 mon.a (mon.0) 1252 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[66542]: audit 2024-03-07T03:30:42.323575+0000 mon.a (mon.0) 1253 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:42 smithi195 bash[66542]: cluster 2024-03-07T03:30:42.338481+0000 mon.a (mon.0) 1254 : cluster 0 osdmap e735: 8 total, 8 up, 8 in 2024-03-07T03:30:44.445 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:30:44.445 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapRemove2 (1978 ms) 2024-03-07T03:30:44.445 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-03-07T03:30:44.445 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:44 smithi148 bash[79819]: cluster 2024-03-07T03:30:42.720906+0000 mgr.y (mgr.34101) 503 : cluster 0 pgmap v857: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-03-07T03:30:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:44 smithi148 bash[79819]: cluster 2024-03-07T03:30:43.342135+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2024-03-07T03:30:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:44 smithi148 bash[82222]: cluster 2024-03-07T03:30:42.720906+0000 mgr.y (mgr.34101) 503 : cluster 0 pgmap v857: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-03-07T03:30:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:44 smithi148 bash[82222]: cluster 2024-03-07T03:30:43.342135+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2024-03-07T03:30:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:44 smithi195 bash[66542]: cluster 2024-03-07T03:30:42.720906+0000 mgr.y (mgr.34101) 503 : cluster 0 pgmap v857: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 2.0 KiB/s wr, 6 op/s 2024-03-07T03:30:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:44 smithi195 bash[66542]: cluster 2024-03-07T03:30:43.342135+0000 mon.a (mon.0) 1255 : cluster 0 osdmap e736: 8 total, 8 up, 8 in 2024-03-07T03:30:45.452 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1007 ms) 2024-03-07T03:30:45.452 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-03-07T03:30:45.452 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:45.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:45 smithi148 bash[79819]: cluster 2024-03-07T03:30:44.350573+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2024-03-07T03:30:45.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:45 smithi148 bash[82222]: cluster 2024-03-07T03:30:44.350573+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2024-03-07T03:30:45.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:45 smithi195 bash[66542]: cluster 2024-03-07T03:30:44.350573+0000 mon.a (mon.0) 1256 : cluster 0 osdmap e737: 8 total, 8 up, 8 in 2024-03-07T03:30:46.468 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (1017 ms) 2024-03-07T03:30:46.469 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-03-07T03:30:46.469 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:46.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:46 smithi148 bash[79819]: cluster 2024-03-07T03:30:44.721950+0000 mgr.y (mgr.34101) 504 : cluster 0 pgmap v860: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 233 op/s 2024-03-07T03:30:46.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:46 smithi148 bash[79819]: cluster 2024-03-07T03:30:45.361268+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2024-03-07T03:30:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:46 smithi148 bash[82222]: cluster 2024-03-07T03:30:44.721950+0000 mgr.y (mgr.34101) 504 : cluster 0 pgmap v860: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 233 op/s 2024-03-07T03:30:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:46 smithi148 bash[82222]: cluster 2024-03-07T03:30:45.361268+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2024-03-07T03:30:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:46 smithi195 bash[66542]: cluster 2024-03-07T03:30:44.721950+0000 mgr.y (mgr.34101) 504 : cluster 0 pgmap v860: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 15 KiB/s wr, 233 op/s 2024-03-07T03:30:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:46 smithi195 bash[66542]: cluster 2024-03-07T03:30:45.361268+0000 mon.a (mon.0) 1257 : cluster 0 osdmap e738: 8 total, 8 up, 8 in 2024-03-07T03:30:47.392 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (923 ms) 2024-03-07T03:30:47.393 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-03-07T03:30:47.393 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:47.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:47 smithi148 bash[79819]: cluster 2024-03-07T03:30:46.361913+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2024-03-07T03:30:47.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:47 smithi148 bash[79819]: cluster 2024-03-07T03:30:47.336998+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2024-03-07T03:30:47.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:30:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:30:47] "GET /metrics HTTP/1.1" 200 75195 "" "Prometheus/2.43.0" 2024-03-07T03:30:47.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:47 smithi148 bash[82222]: cluster 2024-03-07T03:30:46.361913+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2024-03-07T03:30:47.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:47 smithi148 bash[82222]: cluster 2024-03-07T03:30:47.336998+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2024-03-07T03:30:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:47 smithi195 bash[66542]: cluster 2024-03-07T03:30:46.361913+0000 mon.a (mon.0) 1258 : cluster 0 osdmap e739: 8 total, 8 up, 8 in 2024-03-07T03:30:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:47 smithi195 bash[66542]: cluster 2024-03-07T03:30:47.336998+0000 mon.a (mon.0) 1259 : cluster 0 osdmap e740: 8 total, 8 up, 8 in 2024-03-07T03:30:48.584 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1192 ms) 2024-03-07T03:30:48.584 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-03-07T03:30:48.584 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:48.640 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (56 ms) 2024-03-07T03:30:48.640 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-03-07T03:30:48.640 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:48.708 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (68 ms) 2024-03-07T03:30:48.708 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-03-07T03:30:48.708 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:30:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[79819]: cluster 2024-03-07T03:30:46.723104+0000 mgr.y (mgr.34101) 505 : cluster 0 pgmap v863: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 197 KiB/s rd, 16 KiB/s wr, 269 op/s 2024-03-07T03:30:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[79819]: audit 2024-03-07T03:30:47.782105+0000 mon.a (mon.0) 1260 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[79819]: audit 2024-03-07T03:30:47.788040+0000 mon.a (mon.0) 1261 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[79819]: audit 2024-03-07T03:30:48.088763+0000 mon.a (mon.0) 1262 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[79819]: audit 2024-03-07T03:30:48.100392+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[82222]: cluster 2024-03-07T03:30:46.723104+0000 mgr.y (mgr.34101) 505 : cluster 0 pgmap v863: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 197 KiB/s rd, 16 KiB/s wr, 269 op/s 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[82222]: audit 2024-03-07T03:30:47.782105+0000 mon.a (mon.0) 1260 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[82222]: audit 2024-03-07T03:30:47.788040+0000 mon.a (mon.0) 1261 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[82222]: audit 2024-03-07T03:30:48.088763+0000 mon.a (mon.0) 1262 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:48 smithi148 bash[82222]: audit 2024-03-07T03:30:48.100392+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:48 smithi195 bash[66542]: cluster 2024-03-07T03:30:46.723104+0000 mgr.y (mgr.34101) 505 : cluster 0 pgmap v863: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 197 KiB/s rd, 16 KiB/s wr, 269 op/s 2024-03-07T03:30:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:48 smithi195 bash[66542]: audit 2024-03-07T03:30:47.782105+0000 mon.a (mon.0) 1260 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:48 smithi195 bash[66542]: audit 2024-03-07T03:30:47.788040+0000 mon.a (mon.0) 1261 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:48 smithi195 bash[66542]: audit 2024-03-07T03:30:48.088763+0000 mon.a (mon.0) 1262 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:48 smithi195 bash[66542]: audit 2024-03-07T03:30:48.100392+0000 mon.a (mon.0) 1263 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[79819]: cluster 2024-03-07T03:30:48.391904+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[79819]: audit 2024-03-07T03:30:48.739367+0000 mon.a (mon.0) 1265 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[79819]: audit 2024-03-07T03:30:48.740216+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[79819]: audit 2024-03-07T03:30:48.745443+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[82222]: cluster 2024-03-07T03:30:48.391904+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[82222]: audit 2024-03-07T03:30:48.739367+0000 mon.a (mon.0) 1265 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[82222]: audit 2024-03-07T03:30:48.740216+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:30:49.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:49 smithi148 bash[82222]: audit 2024-03-07T03:30:48.745443+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:49 smithi195 bash[66542]: cluster 2024-03-07T03:30:48.391904+0000 mon.a (mon.0) 1264 : cluster 0 osdmap e741: 8 total, 8 up, 8 in 2024-03-07T03:30:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:49 smithi195 bash[66542]: audit 2024-03-07T03:30:48.739367+0000 mon.a (mon.0) 1265 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:30:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:49 smithi195 bash[66542]: audit 2024-03-07T03:30:48.740216+0000 mon.a (mon.0) 1266 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:30:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:49 smithi195 bash[66542]: audit 2024-03-07T03:30:48.745443+0000 mon.a (mon.0) 1267 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:30:50.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:50 smithi148 bash[79819]: cluster 2024-03-07T03:30:48.724508+0000 mgr.y (mgr.34101) 506 : cluster 0 pgmap v866: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-03-07T03:30:50.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:50 smithi148 bash[79819]: cluster 2024-03-07T03:30:49.754289+0000 mon.a (mon.0) 1268 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2024-03-07T03:30:50.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:50 smithi148 bash[82222]: cluster 2024-03-07T03:30:48.724508+0000 mgr.y (mgr.34101) 506 : cluster 0 pgmap v866: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-03-07T03:30:50.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:50 smithi148 bash[82222]: cluster 2024-03-07T03:30:49.754289+0000 mon.a (mon.0) 1268 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2024-03-07T03:30:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:50 smithi195 bash[66542]: cluster 2024-03-07T03:30:48.724508+0000 mgr.y (mgr.34101) 506 : cluster 0 pgmap v866: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 7.5 KiB/s wr, 111 op/s 2024-03-07T03:30:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:50 smithi195 bash[66542]: cluster 2024-03-07T03:30:49.754289+0000 mon.a (mon.0) 1268 : cluster 0 osdmap e742: 8 total, 8 up, 8 in 2024-03-07T03:30:52.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:51 smithi148 bash[79819]: cluster 2024-03-07T03:30:50.726662+0000 mgr.y (mgr.34101) 507 : cluster 0 pgmap v868: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 8.5 KiB/s wr, 173 op/s 2024-03-07T03:30:52.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:51 smithi148 bash[79819]: cluster 2024-03-07T03:30:50.755923+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2024-03-07T03:30:52.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:51 smithi148 bash[82222]: cluster 2024-03-07T03:30:50.726662+0000 mgr.y (mgr.34101) 507 : cluster 0 pgmap v868: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 8.5 KiB/s wr, 173 op/s 2024-03-07T03:30:52.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:51 smithi148 bash[82222]: cluster 2024-03-07T03:30:50.755923+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2024-03-07T03:30:52.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:51 smithi195 bash[66542]: cluster 2024-03-07T03:30:50.726662+0000 mgr.y (mgr.34101) 507 : cluster 0 pgmap v868: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 8.5 KiB/s wr, 173 op/s 2024-03-07T03:30:52.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:51 smithi195 bash[66542]: cluster 2024-03-07T03:30:50.755923+0000 mon.a (mon.0) 1269 : cluster 0 osdmap e743: 8 total, 8 up, 8 in 2024-03-07T03:30:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:30:52 smithi195 bash[59134]: ts=2024-03-07T03:30:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:30:53.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:52 smithi148 bash[82222]: cluster 2024-03-07T03:30:51.766855+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2024-03-07T03:30:53.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:52 smithi148 bash[82222]: cluster 2024-03-07T03:30:52.344331+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2024-03-07T03:30:53.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:52 smithi148 bash[79819]: cluster 2024-03-07T03:30:51.766855+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2024-03-07T03:30:53.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:52 smithi148 bash[79819]: cluster 2024-03-07T03:30:52.344331+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2024-03-07T03:30:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:52 smithi195 bash[66542]: cluster 2024-03-07T03:30:51.766855+0000 mon.a (mon.0) 1270 : cluster 0 osdmap e744: 8 total, 8 up, 8 in 2024-03-07T03:30:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:52 smithi195 bash[66542]: cluster 2024-03-07T03:30:52.344331+0000 mon.a (mon.0) 1271 : cluster 0 osdmap e745: 8 total, 8 up, 8 in 2024-03-07T03:30:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:54 smithi148 bash[79819]: cluster 2024-03-07T03:30:52.727798+0000 mgr.y (mgr.34101) 508 : cluster 0 pgmap v872: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 174 KiB/s rd, 7.7 KiB/s wr, 223 op/s 2024-03-07T03:30:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:54 smithi148 bash[79819]: cluster 2024-03-07T03:30:53.364919+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2024-03-07T03:30:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:54 smithi148 bash[82222]: cluster 2024-03-07T03:30:52.727798+0000 mgr.y (mgr.34101) 508 : cluster 0 pgmap v872: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 174 KiB/s rd, 7.7 KiB/s wr, 223 op/s 2024-03-07T03:30:54.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:54 smithi148 bash[82222]: cluster 2024-03-07T03:30:53.364919+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2024-03-07T03:30:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:54 smithi195 bash[66542]: cluster 2024-03-07T03:30:52.727798+0000 mgr.y (mgr.34101) 508 : cluster 0 pgmap v872: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 174 KiB/s rd, 7.7 KiB/s wr, 223 op/s 2024-03-07T03:30:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:54 smithi195 bash[66542]: cluster 2024-03-07T03:30:53.364919+0000 mon.a (mon.0) 1272 : cluster 0 osdmap e746: 8 total, 8 up, 8 in 2024-03-07T03:30:55.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:55 smithi148 bash[79819]: cluster 2024-03-07T03:30:54.368668+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2024-03-07T03:30:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:55 smithi148 bash[82222]: cluster 2024-03-07T03:30:54.368668+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2024-03-07T03:30:55.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:55 smithi195 bash[66542]: cluster 2024-03-07T03:30:54.368668+0000 mon.a (mon.0) 1273 : cluster 0 osdmap e747: 8 total, 8 up, 8 in 2024-03-07T03:30:56.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:56 smithi148 bash[79819]: cluster 2024-03-07T03:30:54.728963+0000 mgr.y (mgr.34101) 509 : cluster 0 pgmap v875: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 6.3 KiB/s wr, 147 op/s 2024-03-07T03:30:56.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:56 smithi148 bash[79819]: cluster 2024-03-07T03:30:55.390450+0000 mon.a (mon.0) 1274 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2024-03-07T03:30:56.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:56 smithi148 bash[82222]: cluster 2024-03-07T03:30:54.728963+0000 mgr.y (mgr.34101) 509 : cluster 0 pgmap v875: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 6.3 KiB/s wr, 147 op/s 2024-03-07T03:30:56.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:56 smithi148 bash[82222]: cluster 2024-03-07T03:30:55.390450+0000 mon.a (mon.0) 1274 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2024-03-07T03:30:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:56 smithi195 bash[66542]: cluster 2024-03-07T03:30:54.728963+0000 mgr.y (mgr.34101) 509 : cluster 0 pgmap v875: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 6.3 KiB/s wr, 147 op/s 2024-03-07T03:30:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:56 smithi195 bash[66542]: cluster 2024-03-07T03:30:55.390450+0000 mon.a (mon.0) 1274 : cluster 0 osdmap e748: 8 total, 8 up, 8 in 2024-03-07T03:30:57.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:57 smithi148 bash[79819]: cluster 2024-03-07T03:30:56.405219+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2024-03-07T03:30:57.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:57 smithi148 bash[79819]: audit 2024-03-07T03:30:57.324983+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:57.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:57 smithi148 bash[82222]: cluster 2024-03-07T03:30:56.405219+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2024-03-07T03:30:57.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:57 smithi148 bash[82222]: audit 2024-03-07T03:30:57.324983+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:57.542 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:30:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:30:57] "GET /metrics HTTP/1.1" 200 75195 "" "Prometheus/2.43.0" 2024-03-07T03:30:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:57 smithi195 bash[66542]: cluster 2024-03-07T03:30:56.405219+0000 mon.a (mon.0) 1275 : cluster 0 osdmap e749: 8 total, 8 up, 8 in 2024-03-07T03:30:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:57 smithi195 bash[66542]: audit 2024-03-07T03:30:57.324983+0000 mon.a (mon.0) 1276 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:30:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:58 smithi148 bash[79819]: cluster 2024-03-07T03:30:56.730069+0000 mgr.y (mgr.34101) 510 : cluster 0 pgmap v878: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.5 KiB/s wr, 104 op/s 2024-03-07T03:30:58.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:58 smithi148 bash[79819]: cluster 2024-03-07T03:30:57.419436+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2024-03-07T03:30:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:58 smithi148 bash[82222]: cluster 2024-03-07T03:30:56.730069+0000 mgr.y (mgr.34101) 510 : cluster 0 pgmap v878: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.5 KiB/s wr, 104 op/s 2024-03-07T03:30:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:58 smithi148 bash[82222]: cluster 2024-03-07T03:30:57.419436+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2024-03-07T03:30:58.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:58 smithi195 bash[66542]: cluster 2024-03-07T03:30:56.730069+0000 mgr.y (mgr.34101) 510 : cluster 0 pgmap v878: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 6.5 KiB/s wr, 104 op/s 2024-03-07T03:30:58.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:58 smithi195 bash[66542]: cluster 2024-03-07T03:30:57.419436+0000 mon.a (mon.0) 1277 : cluster 0 osdmap e750: 8 total, 8 up, 8 in 2024-03-07T03:30:59.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:30:59 smithi148 bash[79819]: cluster 2024-03-07T03:30:58.422833+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2024-03-07T03:30:59.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:30:59 smithi148 bash[82222]: cluster 2024-03-07T03:30:58.422833+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2024-03-07T03:30:59.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:30:59 smithi195 bash[66542]: cluster 2024-03-07T03:30:58.422833+0000 mon.a (mon.0) 1278 : cluster 0 osdmap e751: 8 total, 8 up, 8 in 2024-03-07T03:31:00.093 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (11385 ms) 2024-03-07T03:31:00.093 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.Metadata 2024-03-07T03:31:00.093 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:00.713 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.Metadata (621 ms) 2024-03-07T03:31:00.714 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-03-07T03:31:00.714 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:00 smithi148 bash[79819]: cluster 2024-03-07T03:30:58.732565+0000 mgr.y (mgr.34101) 511 : cluster 0 pgmap v881: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.2 KiB/s wr, 73 op/s 2024-03-07T03:31:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:00 smithi148 bash[82222]: cluster 2024-03-07T03:30:58.732565+0000 mgr.y (mgr.34101) 511 : cluster 0 pgmap v881: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.2 KiB/s wr, 73 op/s 2024-03-07T03:31:00.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:00 smithi195 bash[66542]: cluster 2024-03-07T03:30:58.732565+0000 mgr.y (mgr.34101) 511 : cluster 0 pgmap v881: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.2 KiB/s wr, 73 op/s 2024-03-07T03:31:01.764 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.MetadataPP (1050 ms) 2024-03-07T03:31:01.764 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-03-07T03:31:01.764 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:01.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:01 smithi148 bash[79819]: cluster 2024-03-07T03:31:00.453193+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2024-03-07T03:31:01.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:01 smithi148 bash[82222]: cluster 2024-03-07T03:31:00.453193+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2024-03-07T03:31:01.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:01 smithi195 bash[66542]: cluster 2024-03-07T03:31:00.453193+0000 mon.a (mon.0) 1279 : cluster 0 osdmap e752: 8 total, 8 up, 8 in 2024-03-07T03:31:02.030 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.UpdateFeatures (266 ms) 2024-03-07T03:31:02.030 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-03-07T03:31:02.030 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-03-07T03:31:02.031 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-03-07T03:31:02.031 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:02 smithi195 bash[66542]: cluster 2024-03-07T03:31:00.733959+0000 mgr.y (mgr.34101) 512 : cluster 0 pgmap v883: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 10 KiB/s wr, 92 op/s 2024-03-07T03:31:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:02 smithi195 bash[66542]: cluster 2024-03-07T03:31:01.479440+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2024-03-07T03:31:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:31:02 smithi195 bash[59134]: ts=2024-03-07T03:31:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:02.672 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:31:02.672 INFO:tasks.workunit.client.0.smithi148.stdout:25.00% done 2024-03-07T03:31:02.672 INFO:tasks.workunit.client.0.smithi148.stdout:50.00% done 2024-03-07T03:31:02.672 INFO:tasks.workunit.client.0.smithi148.stdout:75.00% done 2024-03-07T03:31:02.672 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.RebuildObjectMap (642 ms) 2024-03-07T03:31:02.672 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-03-07T03:31:02.758 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:31:02.759 INFO:tasks.workunit.client.0.smithi148.stdout:25.00% done 2024-03-07T03:31:02.759 INFO:tasks.workunit.client.0.smithi148.stdout:50.00% done 2024-03-07T03:31:02.759 INFO:tasks.workunit.client.0.smithi148.stdout:75.00% done 2024-03-07T03:31:02.759 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (86 ms) 2024-03-07T03:31:02.759 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-03-07T03:31:02.759 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:02.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:02 smithi148 bash[79819]: cluster 2024-03-07T03:31:00.733959+0000 mgr.y (mgr.34101) 512 : cluster 0 pgmap v883: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 10 KiB/s wr, 92 op/s 2024-03-07T03:31:02.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:02 smithi148 bash[79819]: cluster 2024-03-07T03:31:01.479440+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2024-03-07T03:31:02.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:02 smithi148 bash[82222]: cluster 2024-03-07T03:31:00.733959+0000 mgr.y (mgr.34101) 512 : cluster 0 pgmap v883: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 10 KiB/s wr, 92 op/s 2024-03-07T03:31:02.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:02 smithi148 bash[82222]: cluster 2024-03-07T03:31:01.479440+0000 mon.a (mon.0) 1280 : cluster 0 osdmap e753: 8 total, 8 up, 8 in 2024-03-07T03:31:03.691 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:31:03.691 INFO:tasks.workunit.client.0.smithi148.stdout:25.00% done 2024-03-07T03:31:03.691 INFO:tasks.workunit.client.0.smithi148.stdout:50.00% done 2024-03-07T03:31:03.691 INFO:tasks.workunit.client.0.smithi148.stdout:75.00% done 2024-03-07T03:31:03.691 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CheckObjectMap (933 ms) 2024-03-07T03:31:03.691 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-03-07T03:31:03.691 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:03.763 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.BlockingAIO (73 ms) 2024-03-07T03:31:03.764 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-03-07T03:31:03.764 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:03.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:03 smithi148 bash[79819]: cluster 2024-03-07T03:31:02.480935+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2024-03-07T03:31:03.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:03 smithi148 bash[82222]: cluster 2024-03-07T03:31:02.480935+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2024-03-07T03:31:03.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:03 smithi195 bash[66542]: cluster 2024-03-07T03:31:02.480935+0000 mon.a (mon.0) 1281 : cluster 0 osdmap e754: 8 total, 8 up, 8 in 2024-03-07T03:31:04.026 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (262 ms) 2024-03-07T03:31:04.026 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-03-07T03:31:04.026 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_librbd.cc:10118: Skipped 2024-03-07T03:31:04.026 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:31:04.026 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-03-07T03:31:04.026 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-03-07T03:31:04.026 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:04.692 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (666 ms) 2024-03-07T03:31:04.692 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-03-07T03:31:04.692 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:04 smithi148 bash[79819]: cluster 2024-03-07T03:31:02.735106+0000 mgr.y (mgr.34101) 513 : cluster 0 pgmap v886: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 23 KiB/s wr, 106 op/s 2024-03-07T03:31:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:04 smithi148 bash[79819]: cluster 2024-03-07T03:31:03.508109+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2024-03-07T03:31:04.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:04 smithi148 bash[82222]: cluster 2024-03-07T03:31:02.735106+0000 mgr.y (mgr.34101) 513 : cluster 0 pgmap v886: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 23 KiB/s wr, 106 op/s 2024-03-07T03:31:04.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:04 smithi148 bash[82222]: cluster 2024-03-07T03:31:03.508109+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2024-03-07T03:31:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:04 smithi195 bash[66542]: cluster 2024-03-07T03:31:02.735106+0000 mgr.y (mgr.34101) 513 : cluster 0 pgmap v886: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 23 KiB/s wr, 106 op/s 2024-03-07T03:31:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:04 smithi195 bash[66542]: cluster 2024-03-07T03:31:03.508109+0000 mon.a (mon.0) 1282 : cluster 0 osdmap e755: 8 total, 8 up, 8 in 2024-03-07T03:31:05.607 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (915 ms) 2024-03-07T03:31:05.607 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-03-07T03:31:05.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:05 smithi148 bash[79819]: cluster 2024-03-07T03:31:04.508570+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2024-03-07T03:31:05.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:05 smithi148 bash[82222]: cluster 2024-03-07T03:31:04.508570+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2024-03-07T03:31:05.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:05 smithi195 bash[66542]: cluster 2024-03-07T03:31:04.508570+0000 mon.a (mon.0) 1283 : cluster 0 osdmap e756: 8 total, 8 up, 8 in 2024-03-07T03:31:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:06 smithi148 bash[82222]: cluster 2024-03-07T03:31:04.736270+0000 mgr.y (mgr.34101) 514 : cluster 0 pgmap v889: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 675 KiB/s rd, 218 KiB/s wr, 1.03k op/s 2024-03-07T03:31:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:06 smithi148 bash[82222]: cluster 2024-03-07T03:31:05.536953+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2024-03-07T03:31:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:06 smithi148 bash[79819]: cluster 2024-03-07T03:31:04.736270+0000 mgr.y (mgr.34101) 514 : cluster 0 pgmap v889: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 675 KiB/s rd, 218 KiB/s wr, 1.03k op/s 2024-03-07T03:31:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:06 smithi148 bash[79819]: cluster 2024-03-07T03:31:05.536953+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2024-03-07T03:31:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:06 smithi195 bash[66542]: cluster 2024-03-07T03:31:04.736270+0000 mgr.y (mgr.34101) 514 : cluster 0 pgmap v889: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 675 KiB/s rd, 218 KiB/s wr, 1.03k op/s 2024-03-07T03:31:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:06 smithi195 bash[66542]: cluster 2024-03-07T03:31:05.536953+0000 mon.a (mon.0) 1284 : cluster 0 osdmap e757: 8 total, 8 up, 8 in 2024-03-07T03:31:06.823 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:31:06.823 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:31:06.823 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestImageOptions (1216 ms) 2024-03-07T03:31:06.823 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-03-07T03:31:07.537 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:31:07] "GET /metrics HTTP/1.1" 200 75190 "" "Prometheus/2.43.0" 2024-03-07T03:31:07.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:07 smithi148 bash[79819]: cluster 2024-03-07T03:31:06.537180+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2024-03-07T03:31:07.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:07 smithi148 bash[82222]: cluster 2024-03-07T03:31:06.537180+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2024-03-07T03:31:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:07 smithi195 bash[66542]: cluster 2024-03-07T03:31:06.537180+0000 mon.a (mon.0) 1285 : cluster 0 osdmap e758: 8 total, 8 up, 8 in 2024-03-07T03:31:07.844 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:100.00% done 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (1022 ms) 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-03-07T03:31:07.845 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:wrote: 512 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:31:07.942 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:31:07.943 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:started write 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:write completion cb called! 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 0 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:finished write 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:31:07.944 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:created completion 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:started read 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:read completion cb called! 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:return value is: 512 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ImagePollIO (97 ms) 2024-03-07T03:31:07.945 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.Mirror 2024-03-07T03:31:07.954 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:07.976 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:08 smithi148 bash[82222]: cluster 2024-03-07T03:31:06.737461+0000 mgr.y (mgr.34101) 515 : cluster 0 pgmap v892: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 722 KiB/s rd, 234 KiB/s wr, 1.07k op/s 2024-03-07T03:31:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:08 smithi148 bash[82222]: cluster 2024-03-07T03:31:07.556576+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2024-03-07T03:31:08.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:08 smithi148 bash[79819]: cluster 2024-03-07T03:31:06.737461+0000 mgr.y (mgr.34101) 515 : cluster 0 pgmap v892: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 722 KiB/s rd, 234 KiB/s wr, 1.07k op/s 2024-03-07T03:31:08.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:08 smithi148 bash[79819]: cluster 2024-03-07T03:31:07.556576+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2024-03-07T03:31:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:08 smithi195 bash[66542]: cluster 2024-03-07T03:31:06.737461+0000 mgr.y (mgr.34101) 515 : cluster 0 pgmap v892: 617 pgs: 617 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 722 KiB/s rd, 234 KiB/s wr, 1.07k op/s 2024-03-07T03:31:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:08 smithi195 bash[66542]: cluster 2024-03-07T03:31:07.556576+0000 mon.a (mon.0) 1286 : cluster 0 osdmap e759: 8 total, 8 up, 8 in 2024-03-07T03:31:08.963 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.Mirror (1021 ms) 2024-03-07T03:31:08.963 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-03-07T03:31:09.011 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (48 ms) 2024-03-07T03:31:09.011 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-03-07T03:31:10.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: cluster 2024-03-07T03:31:08.573413+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2024-03-07T03:31:10.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.904367+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:31:10.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.905062+0000 mon.a (mon.0) 1288 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:31:10.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.911195+0000 mon.a (mon.0) 1289 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-07T03:31:10.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.916628+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.917276+0000 mon.a (mon.0) 1290 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.923315+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]': finished 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.942310+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.942954+0000 mon.a (mon.0) 1292 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.948565+0000 mon.a (mon.0) 1293 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]': finished 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.954660+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.955396+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.961213+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]': finished 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.987387+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:08.996616+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:09.000104+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:09.000930+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:09.001495+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[82222]: audit 2024-03-07T03:31:09.008830+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]': finished 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: cluster 2024-03-07T03:31:08.573413+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.904367+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.905062+0000 mon.a (mon.0) 1288 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.911195+0000 mon.a (mon.0) 1289 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.916628+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]: dispatch 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.917276+0000 mon.a (mon.0) 1290 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]: dispatch 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.923315+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]': finished 2024-03-07T03:31:10.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.942310+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.942954+0000 mon.a (mon.0) 1292 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.948565+0000 mon.a (mon.0) 1293 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]': finished 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.954660+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.955396+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.961213+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]': finished 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.987387+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:08.996616+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:09.000104+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:09.000930+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:09.001495+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:09 smithi148 bash[79819]: audit 2024-03-07T03:31:09.008830+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]': finished 2024-03-07T03:31:10.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: cluster 2024-03-07T03:31:08.573413+0000 mon.a (mon.0) 1287 : cluster 0 osdmap e760: 8 total, 8 up, 8 in 2024-03-07T03:31:10.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.904367+0000 mon.c (mon.2) 70 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:31:10.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.905062+0000 mon.a (mon.0) 1288 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]: dispatch 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.911195+0000 mon.a (mon.0) 1289 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.916628+0000 mon.c (mon.2) 71 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]: dispatch 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.917276+0000 mon.a (mon.0) 1290 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]: dispatch 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.923315+0000 mon.a (mon.0) 1291 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/cfecf052-f5ab-4117-91a5-fe69e76eca77"}]': finished 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.942310+0000 mon.c (mon.2) 72 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]: dispatch 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.942954+0000 mon.a (mon.0) 1292 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]: dispatch 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.948565+0000 mon.a (mon.0) 1293 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4ca450c0-8ae8-4484-9a1b-d3016033e253"}]': finished 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.954660+0000 mon.c (mon.2) 73 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]: dispatch 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.955396+0000 mon.a (mon.0) 1294 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]: dispatch 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.961213+0000 mon.a (mon.0) 1295 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/5eb0a009-501e-4431-a7f9-dc64abb67ae6"}]': finished 2024-03-07T03:31:10.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.987387+0000 mon.c (mon.2) 74 : audit 0 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:08.996616+0000 mon.a (mon.0) 1296 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:31:10.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:09.000104+0000 mon.c (mon.2) 75 : audit 0 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key get", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:09.000930+0000 mon.c (mon.2) 76 : audit 1 from='client.? 172.21.15.148:0/2472687728' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:09.001495+0000 mon.a (mon.0) 1297 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]: dispatch 2024-03-07T03:31:10.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:09 smithi195 bash[66542]: audit 2024-03-07T03:31:09.008830+0000 mon.a (mon.0) 1298 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/978d11a4-f3aa-4f86-9c4d-2caf9b313bc2"}]': finished 2024-03-07T03:31:11.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:10 smithi148 bash[79819]: cluster 2024-03-07T03:31:08.738237+0000 mgr.y (mgr.34101) 516 : cluster 0 pgmap v895: 617 pgs: 617 active+clean; 309 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 173 KiB/s rd, 1.0 MiB/s wr, 218 op/s 2024-03-07T03:31:11.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:10 smithi148 bash[79819]: cluster 2024-03-07T03:31:10.017147+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2024-03-07T03:31:11.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:10 smithi148 bash[82222]: cluster 2024-03-07T03:31:08.738237+0000 mgr.y (mgr.34101) 516 : cluster 0 pgmap v895: 617 pgs: 617 active+clean; 309 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 173 KiB/s rd, 1.0 MiB/s wr, 218 op/s 2024-03-07T03:31:11.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:10 smithi148 bash[82222]: cluster 2024-03-07T03:31:10.017147+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2024-03-07T03:31:11.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:10 smithi195 bash[66542]: cluster 2024-03-07T03:31:08.738237+0000 mgr.y (mgr.34101) 516 : cluster 0 pgmap v895: 617 pgs: 617 active+clean; 309 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 173 KiB/s rd, 1.0 MiB/s wr, 218 op/s 2024-03-07T03:31:11.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:10 smithi195 bash[66542]: cluster 2024-03-07T03:31:10.017147+0000 mon.a (mon.0) 1299 : cluster 0 osdmap e761: 8 total, 8 up, 8 in 2024-03-07T03:31:12.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:12 smithi148 bash[79819]: cluster 2024-03-07T03:31:10.739983+0000 mgr.y (mgr.34101) 517 : cluster 0 pgmap v897: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 331 KiB/s rd, 1.9 MiB/s wr, 452 op/s 2024-03-07T03:31:12.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:12 smithi148 bash[79819]: cluster 2024-03-07T03:31:11.020890+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2024-03-07T03:31:12.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:12 smithi148 bash[82222]: cluster 2024-03-07T03:31:10.739983+0000 mgr.y (mgr.34101) 517 : cluster 0 pgmap v897: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 331 KiB/s rd, 1.9 MiB/s wr, 452 op/s 2024-03-07T03:31:12.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:12 smithi148 bash[82222]: cluster 2024-03-07T03:31:11.020890+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2024-03-07T03:31:12.304 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:12 smithi195 bash[66542]: cluster 2024-03-07T03:31:10.739983+0000 mgr.y (mgr.34101) 517 : cluster 0 pgmap v897: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 331 KiB/s rd, 1.9 MiB/s wr, 452 op/s 2024-03-07T03:31:12.304 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:12 smithi195 bash[66542]: cluster 2024-03-07T03:31:11.020890+0000 mon.a (mon.0) 1300 : cluster 0 osdmap e762: 8 total, 8 up, 8 in 2024-03-07T03:31:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:31:12 smithi195 bash[59134]: ts=2024-03-07T03:31:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:13.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:13 smithi148 bash[79819]: cluster 2024-03-07T03:31:12.037600+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2024-03-07T03:31:13.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:13 smithi148 bash[79819]: audit 2024-03-07T03:31:12.325532+0000 mon.a (mon.0) 1302 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:13 smithi148 bash[82222]: cluster 2024-03-07T03:31:12.037600+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2024-03-07T03:31:13.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:13 smithi148 bash[82222]: audit 2024-03-07T03:31:12.325532+0000 mon.a (mon.0) 1302 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:13 smithi195 bash[66542]: cluster 2024-03-07T03:31:12.037600+0000 mon.a (mon.0) 1301 : cluster 0 osdmap e763: 8 total, 8 up, 8 in 2024-03-07T03:31:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:13 smithi195 bash[66542]: audit 2024-03-07T03:31:12.325532+0000 mon.a (mon.0) 1302 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:14.134 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (5123 ms) 2024-03-07T03:31:14.134 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-03-07T03:31:14.134 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:14.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:14 smithi148 bash[79819]: cluster 2024-03-07T03:31:12.741081+0000 mgr.y (mgr.34101) 518 : cluster 0 pgmap v900: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 1.1 MiB/s wr, 327 op/s 2024-03-07T03:31:14.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:14 smithi148 bash[79819]: cluster 2024-03-07T03:31:13.049335+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2024-03-07T03:31:14.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:14 smithi148 bash[82222]: cluster 2024-03-07T03:31:12.741081+0000 mgr.y (mgr.34101) 518 : cluster 0 pgmap v900: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 1.1 MiB/s wr, 327 op/s 2024-03-07T03:31:14.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:14 smithi148 bash[82222]: cluster 2024-03-07T03:31:13.049335+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2024-03-07T03:31:14.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:14 smithi195 bash[66542]: cluster 2024-03-07T03:31:12.741081+0000 mgr.y (mgr.34101) 518 : cluster 0 pgmap v900: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 229 KiB/s rd, 1.1 MiB/s wr, 327 op/s 2024-03-07T03:31:14.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:14 smithi195 bash[66542]: cluster 2024-03-07T03:31:13.049335+0000 mon.a (mon.0) 1303 : cluster 0 osdmap e764: 8 total, 8 up, 8 in 2024-03-07T03:31:15.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:15 smithi195 bash[66542]: cluster 2024-03-07T03:31:14.058013+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2024-03-07T03:31:15.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:15 smithi148 bash[79819]: cluster 2024-03-07T03:31:14.058013+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2024-03-07T03:31:15.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:15 smithi148 bash[82222]: cluster 2024-03-07T03:31:14.058013+0000 mon.a (mon.0) 1304 : cluster 0 osdmap e765: 8 total, 8 up, 8 in 2024-03-07T03:31:16.149 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (2015 ms) 2024-03-07T03:31:16.149 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-03-07T03:31:16.149 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:16.280 INFO:tasks.workunit.client.0.smithi148.stdout:1: acquiring exclusive lock 2024-03-07T03:31:16.281 INFO:tasks.workunit.client.0.smithi148.stdout:2: acquiring exclusive lock 2024-03-07T03:31:16.289 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock acquired 2024-03-07T03:31:16.292 INFO:tasks.workunit.client.0.smithi148.stdout:2: releasing exclusive lock 2024-03-07T03:31:16.300 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock released 2024-03-07T03:31:16.300 INFO:tasks.workunit.client.0.smithi148.stdout:2: acquiring exclusive lock 2024-03-07T03:31:16.311 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock acquired 2024-03-07T03:31:16.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:16 smithi195 bash[66542]: cluster 2024-03-07T03:31:14.743005+0000 mgr.y (mgr.34101) 519 : cluster 0 pgmap v903: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 166 KiB/s rd, 141 KiB/s wr, 213 op/s 2024-03-07T03:31:16.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:16 smithi195 bash[66542]: cluster 2024-03-07T03:31:15.067646+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2024-03-07T03:31:16.318 INFO:tasks.workunit.client.0.smithi148.stdout:2: releasing exclusive lock 2024-03-07T03:31:16.325 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock released 2024-03-07T03:31:16.325 INFO:tasks.workunit.client.0.smithi148.stdout:2: acquiring exclusive lock 2024-03-07T03:31:16.334 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock acquired 2024-03-07T03:31:16.375 INFO:tasks.workunit.client.0.smithi148.stdout:1: releasing exclusive lock 2024-03-07T03:31:16.382 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock released 2024-03-07T03:31:16.382 INFO:tasks.workunit.client.0.smithi148.stdout:1: acquiring exclusive lock 2024-03-07T03:31:16.391 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock acquired 2024-03-07T03:31:16.410 INFO:tasks.workunit.client.0.smithi148.stdout:2: releasing exclusive lock 2024-03-07T03:31:16.417 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock released 2024-03-07T03:31:16.417 INFO:tasks.workunit.client.0.smithi148.stdout:2: acquiring exclusive lock 2024-03-07T03:31:16.425 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock acquired 2024-03-07T03:31:16.474 INFO:tasks.workunit.client.0.smithi148.stdout:1: releasing exclusive lock 2024-03-07T03:31:16.483 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock released 2024-03-07T03:31:16.483 INFO:tasks.workunit.client.0.smithi148.stdout:1: acquiring exclusive lock 2024-03-07T03:31:16.493 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock acquired 2024-03-07T03:31:16.496 INFO:tasks.workunit.client.0.smithi148.stdout:2: releasing exclusive lock 2024-03-07T03:31:16.503 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock released 2024-03-07T03:31:16.503 INFO:tasks.workunit.client.0.smithi148.stdout:2: acquiring exclusive lock 2024-03-07T03:31:16.513 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock acquired 2024-03-07T03:31:16.530 INFO:tasks.workunit.client.0.smithi148.stdout:2: releasing exclusive lock 2024-03-07T03:31:16.537 INFO:tasks.workunit.client.0.smithi148.stdout:2: exclusive lock released 2024-03-07T03:31:16.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:16 smithi148 bash[79819]: cluster 2024-03-07T03:31:14.743005+0000 mgr.y (mgr.34101) 519 : cluster 0 pgmap v903: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 166 KiB/s rd, 141 KiB/s wr, 213 op/s 2024-03-07T03:31:16.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:16 smithi148 bash[79819]: cluster 2024-03-07T03:31:15.067646+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2024-03-07T03:31:16.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:16 smithi148 bash[82222]: cluster 2024-03-07T03:31:14.743005+0000 mgr.y (mgr.34101) 519 : cluster 0 pgmap v903: 617 pgs: 617 active+clean; 313 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 166 KiB/s rd, 141 KiB/s wr, 213 op/s 2024-03-07T03:31:16.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:16 smithi148 bash[82222]: cluster 2024-03-07T03:31:15.067646+0000 mon.a (mon.0) 1305 : cluster 0 osdmap e766: 8 total, 8 up, 8 in 2024-03-07T03:31:16.548 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock acquired 2024-03-07T03:31:16.558 INFO:tasks.workunit.client.0.smithi148.stdout:1: releasing exclusive lock 2024-03-07T03:31:16.564 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock released 2024-03-07T03:31:16.565 INFO:tasks.workunit.client.0.smithi148.stdout:1: acquiring exclusive lock 2024-03-07T03:31:16.574 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock acquired 2024-03-07T03:31:16.577 INFO:tasks.workunit.client.0.smithi148.stdout:1: releasing exclusive lock 2024-03-07T03:31:16.586 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock released 2024-03-07T03:31:16.586 INFO:tasks.workunit.client.0.smithi148.stdout:1: acquiring exclusive lock 2024-03-07T03:31:16.596 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock acquired 2024-03-07T03:31:16.631 INFO:tasks.workunit.client.0.smithi148.stdout:1: releasing exclusive lock 2024-03-07T03:31:16.638 INFO:tasks.workunit.client.0.smithi148.stdout:1: exclusive lock released 2024-03-07T03:31:16.676 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ExclusiveLock (527 ms) 2024-03-07T03:31:16.676 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.BreakLock 2024-03-07T03:31:16.711 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:17.190 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.BreakLock (514 ms) 2024-03-07T03:31:17.190 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-03-07T03:31:17.190 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:17.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (68 ms) 2024-03-07T03:31:17.258 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-03-07T03:31:17.259 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-03-07T03:31:17.259 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-03-07T03:31:17.259 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:17.445 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:31:17.445 INFO:tasks.workunit.client.0.smithi148.stdout:1.56% done 2024-03-07T03:31:17.445 INFO:tasks.workunit.client.0.smithi148.stdout:3.12% done 2024-03-07T03:31:17.445 INFO:tasks.workunit.client.0.smithi148.stdout:4.69% done 2024-03-07T03:31:17.445 INFO:tasks.workunit.client.0.smithi148.stdout:6.25% done 2024-03-07T03:31:17.445 INFO:tasks.workunit.client.0.smithi148.stdout:7.81% done 2024-03-07T03:31:17.445 INFO:tasks.workunit.client.0.smithi148.stdout:9.38% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:10.94% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:12.50% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:14.06% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:15.62% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:17.19% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:18.75% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:20.31% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:21.88% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:23.44% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:25.00% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:26.56% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:28.12% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:29.69% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:31.25% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:32.81% done 2024-03-07T03:31:17.446 INFO:tasks.workunit.client.0.smithi148.stdout:34.38% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:35.94% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:37.50% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:39.06% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:40.62% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:42.19% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:43.75% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:45.31% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:46.88% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:48.44% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:50.00% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:51.56% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:53.12% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:54.69% done 2024-03-07T03:31:17.447 INFO:tasks.workunit.client.0.smithi148.stdout:56.25% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:57.81% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:59.38% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:60.94% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:62.50% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:64.06% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:65.62% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:67.19% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:68.75% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:70.31% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:71.88% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:73.44% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:75.00% done 2024-03-07T03:31:17.448 INFO:tasks.workunit.client.0.smithi148.stdout:76.56% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:78.12% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:79.69% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:81.25% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:82.81% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:84.38% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:85.94% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:87.50% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:89.06% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:90.62% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:92.19% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:93.75% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:95.31% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:96.88% done 2024-03-07T03:31:17.449 INFO:tasks.workunit.client.0.smithi148.stdout:98.44% done 2024-03-07T03:31:17.450 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (187 ms) 2024-03-07T03:31:17.450 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-03-07T03:31:17.450 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:17 smithi148 bash[82222]: cluster 2024-03-07T03:31:16.077602+0000 mon.a (mon.0) 1306 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2024-03-07T03:31:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:17 smithi148 bash[82222]: audit 2024-03-07T03:31:16.791999+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]: dispatch 2024-03-07T03:31:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:17 smithi148 bash[82222]: audit 2024-03-07T03:31:16.792467+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]: dispatch 2024-03-07T03:31:17.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:31:17] "GET /metrics HTTP/1.1" 200 75196 "" "Prometheus/2.43.0" 2024-03-07T03:31:17.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:17 smithi148 bash[79819]: cluster 2024-03-07T03:31:16.077602+0000 mon.a (mon.0) 1306 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2024-03-07T03:31:17.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:17 smithi148 bash[79819]: audit 2024-03-07T03:31:16.791999+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]: dispatch 2024-03-07T03:31:17.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:17 smithi148 bash[79819]: audit 2024-03-07T03:31:16.792467+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]: dispatch 2024-03-07T03:31:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:17 smithi195 bash[66542]: cluster 2024-03-07T03:31:16.077602+0000 mon.a (mon.0) 1306 : cluster 0 osdmap e767: 8 total, 8 up, 8 in 2024-03-07T03:31:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:17 smithi195 bash[66542]: audit 2024-03-07T03:31:16.791999+0000 mon.c (mon.2) 77 : audit 1 from='client.? 172.21.15.148:0/1064018435' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]: dispatch 2024-03-07T03:31:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:17 smithi195 bash[66542]: audit 2024-03-07T03:31:16.792467+0000 mon.a (mon.0) 1307 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]: dispatch 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:0.00% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:1.56% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:3.12% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:4.69% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:6.25% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:7.81% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:9.38% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:10.94% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:12.50% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:14.06% done 2024-03-07T03:31:17.576 INFO:tasks.workunit.client.0.smithi148.stdout:15.62% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:17.19% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:18.75% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:20.31% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:21.88% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:23.44% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:25.00% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:26.56% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:28.12% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:29.69% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:31.25% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:32.81% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:34.38% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:35.94% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:37.50% done 2024-03-07T03:31:17.577 INFO:tasks.workunit.client.0.smithi148.stdout:39.06% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:40.62% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:42.19% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:43.75% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:45.31% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:46.88% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:48.44% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:50.00% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:51.56% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:53.12% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:54.69% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:56.25% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:57.81% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:59.38% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:60.94% done 2024-03-07T03:31:17.578 INFO:tasks.workunit.client.0.smithi148.stdout:62.50% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:64.06% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:65.62% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:67.19% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:68.75% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:70.31% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:71.88% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:73.44% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:75.00% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:76.56% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:78.12% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:79.69% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:81.25% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:82.81% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:84.38% done 2024-03-07T03:31:17.579 INFO:tasks.workunit.client.0.smithi148.stdout:85.94% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:87.50% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:89.06% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:90.62% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:92.19% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:93.75% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:95.31% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:96.88% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:98.44% done 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (131 ms) 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-03-07T03:31:17.580 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:17.596 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:17.929 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestTrashPurge (353 ms) 2024-03-07T03:31:17.929 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-03-07T03:31:17.929 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:18.056 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (127 ms) 2024-03-07T03:31:18.056 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-03-07T03:31:18.057 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:18.155 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestListWatchers (99 ms) 2024-03-07T03:31:18.155 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-03-07T03:31:18.155 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[79819]: cluster 2024-03-07T03:31:16.744110+0000 mgr.y (mgr.34101) 520 : cluster 0 pgmap v906: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 79 KiB/s wr, 261 op/s 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[79819]: audit 2024-03-07T03:31:17.080493+0000 mon.a (mon.0) 1308 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]': finished 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[79819]: cluster 2024-03-07T03:31:17.087296+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[79819]: cluster 2024-03-07T03:31:17.352976+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[82222]: cluster 2024-03-07T03:31:16.744110+0000 mgr.y (mgr.34101) 520 : cluster 0 pgmap v906: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 79 KiB/s wr, 261 op/s 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[82222]: audit 2024-03-07T03:31:17.080493+0000 mon.a (mon.0) 1308 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]': finished 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[82222]: cluster 2024-03-07T03:31:17.087296+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2024-03-07T03:31:18.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:18 smithi148 bash[82222]: cluster 2024-03-07T03:31:17.352976+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2024-03-07T03:31:18.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:18 smithi195 bash[66542]: cluster 2024-03-07T03:31:16.744110+0000 mgr.y (mgr.34101) 520 : cluster 0 pgmap v906: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 79 KiB/s wr, 261 op/s 2024-03-07T03:31:18.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:18 smithi195 bash[66542]: audit 2024-03-07T03:31:17.080493+0000 mon.a (mon.0) 1308 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.148:0/2471490164"}]': finished 2024-03-07T03:31:18.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:18 smithi195 bash[66542]: cluster 2024-03-07T03:31:17.087296+0000 mon.a (mon.0) 1309 : cluster 0 osdmap e768: 8 total, 8 up, 8 in 2024-03-07T03:31:18.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:18 smithi195 bash[66542]: cluster 2024-03-07T03:31:17.352976+0000 mon.a (mon.0) 1310 : cluster 0 osdmap e769: 8 total, 8 up, 8 in 2024-03-07T03:31:19.161 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestSetSnapById (1006 ms) 2024-03-07T03:31:19.161 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.Namespaces 2024-03-07T03:31:19.214 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.Namespaces (54 ms) 2024-03-07T03:31:19.215 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-03-07T03:31:19.287 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:19.298 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:19.480 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.NamespacesPP (265 ms) 2024-03-07T03:31:19.480 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.Migration 2024-03-07T03:31:19.480 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:19.480 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:20.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:20 smithi148 bash[79819]: cluster 2024-03-07T03:31:18.746677+0000 mgr.y (mgr.34101) 521 : cluster 0 pgmap v909: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 143 KiB/s wr, 224 op/s 2024-03-07T03:31:20.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:20 smithi148 bash[79819]: cluster 2024-03-07T03:31:19.098064+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2024-03-07T03:31:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:20 smithi148 bash[82222]: cluster 2024-03-07T03:31:18.746677+0000 mgr.y (mgr.34101) 521 : cluster 0 pgmap v909: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 143 KiB/s wr, 224 op/s 2024-03-07T03:31:20.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:20 smithi148 bash[82222]: cluster 2024-03-07T03:31:19.098064+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2024-03-07T03:31:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:20 smithi195 bash[66542]: cluster 2024-03-07T03:31:18.746677+0000 mgr.y (mgr.34101) 521 : cluster 0 pgmap v909: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 156 KiB/s rd, 143 KiB/s wr, 224 op/s 2024-03-07T03:31:20.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:20 smithi195 bash[66542]: cluster 2024-03-07T03:31:19.098064+0000 mon.a (mon.0) 1311 : cluster 0 osdmap e770: 8 total, 8 up, 8 in 2024-03-07T03:31:20.639 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.Migration (1159 ms) 2024-03-07T03:31:20.639 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-03-07T03:31:20.639 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:20.639 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:21.822 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.MigrationPP (1183 ms) 2024-03-07T03:31:21.822 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-03-07T03:31:21.822 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:21.868 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (46 ms) 2024-03-07T03:31:21.868 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-03-07T03:31:21.868 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:21.916 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (48 ms) 2024-03-07T03:31:21.916 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-03-07T03:31:21.916 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:22.365 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (449 ms) 2024-03-07T03:31:22.365 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-03-07T03:31:22.407 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[79819]: cluster 2024-03-07T03:31:20.749732+0000 mgr.y (mgr.34101) 522 : cluster 0 pgmap v911: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 319 KiB/s rd, 74 KiB/s wr, 371 op/s 2024-03-07T03:31:22.408 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[82222]: cluster 2024-03-07T03:31:20.749732+0000 mgr.y (mgr.34101) 522 : cluster 0 pgmap v911: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 319 KiB/s rd, 74 KiB/s wr, 371 op/s 2024-03-07T03:31:22.425 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.PoolMetadata (60 ms) 2024-03-07T03:31:22.425 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-03-07T03:31:22.453 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.PoolMetadataPP (28 ms) 2024-03-07T03:31:22.453 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.Config 2024-03-07T03:31:22.460 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:22.551 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.Config (99 ms) 2024-03-07T03:31:22.552 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-03-07T03:31:22.557 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:31:22 smithi195 bash[59134]: ts=2024-03-07T03:31:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:22 smithi195 bash[66542]: cluster 2024-03-07T03:31:20.749732+0000 mgr.y (mgr.34101) 522 : cluster 0 pgmap v911: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 319 KiB/s rd, 74 KiB/s wr, 371 op/s 2024-03-07T03:31:22.624 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ConfigPP (72 ms) 2024-03-07T03:31:22.624 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-03-07T03:31:22.791 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[72584]: [07/Mar/2024:03:31:22] ENGINE Bus STOPPING 2024-03-07T03:31:22.791 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[72584]: [07/Mar/2024:03:31:22] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:31:22.791 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[72584]: [07/Mar/2024:03:31:22] ENGINE Bus STOPPED 2024-03-07T03:31:22.791 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[72584]: [07/Mar/2024:03:31:22] ENGINE Bus STARTING 2024-03-07T03:31:22.791 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[72584]: [07/Mar/2024:03:31:22] ENGINE Serving on http://:::9283 2024-03-07T03:31:22.791 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:22 smithi148 bash[72584]: [07/Mar/2024:03:31:22] ENGINE Bus STARTED 2024-03-07T03:31:23.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[79819]: cluster 2024-03-07T03:31:22.116853+0000 mon.a (mon.0) 1312 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2024-03-07T03:31:23.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[79819]: cluster 2024-03-07T03:31:22.349576+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2024-03-07T03:31:23.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[79819]: audit 2024-03-07T03:31:22.407405+0000 mon.a (mon.0) 1314 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:31:23.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[79819]: audit 2024-03-07T03:31:22.462348+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:31:23.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[79819]: audit 2024-03-07T03:31:22.464215+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:31:23.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[79819]: audit 2024-03-07T03:31:22.465102+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:31:23.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[79819]: audit 2024-03-07T03:31:22.470921+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:23.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[82222]: cluster 2024-03-07T03:31:22.116853+0000 mon.a (mon.0) 1312 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2024-03-07T03:31:23.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[82222]: cluster 2024-03-07T03:31:22.349576+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2024-03-07T03:31:23.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[82222]: audit 2024-03-07T03:31:22.407405+0000 mon.a (mon.0) 1314 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:31:23.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[82222]: audit 2024-03-07T03:31:22.462348+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:31:23.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[82222]: audit 2024-03-07T03:31:22.464215+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:31:23.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[82222]: audit 2024-03-07T03:31:22.465102+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:31:23.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:23 smithi148 bash[82222]: audit 2024-03-07T03:31:22.470921+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:23.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:23 smithi195 bash[66542]: cluster 2024-03-07T03:31:22.116853+0000 mon.a (mon.0) 1312 : cluster 0 osdmap e771: 8 total, 8 up, 8 in 2024-03-07T03:31:23.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:23 smithi195 bash[66542]: cluster 2024-03-07T03:31:22.349576+0000 mon.a (mon.0) 1313 : cluster 0 osdmap e772: 8 total, 8 up, 8 in 2024-03-07T03:31:23.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:23 smithi195 bash[66542]: audit 2024-03-07T03:31:22.407405+0000 mon.a (mon.0) 1314 : audit 1 from='client.? ' entity='client.admin' 2024-03-07T03:31:23.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:23 smithi195 bash[66542]: audit 2024-03-07T03:31:22.462348+0000 mon.a (mon.0) 1315 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:31:23.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:23 smithi195 bash[66542]: audit 2024-03-07T03:31:22.464215+0000 mon.a (mon.0) 1316 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:31:23.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:23 smithi195 bash[66542]: audit 2024-03-07T03:31:22.465102+0000 mon.a (mon.0) 1317 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:31:23.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:23 smithi195 bash[66542]: audit 2024-03-07T03:31:22.470921+0000 mon.a (mon.0) 1318 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:24.501 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[79819]: cluster 2024-03-07T03:31:22.750785+0000 mgr.y (mgr.34101) 523 : cluster 0 pgmap v914: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 326 KiB/s rd, 66 KiB/s wr, 386 op/s 2024-03-07T03:31:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[79819]: cluster 2024-03-07T03:31:23.489544+0000 mon.a (mon.0) 1319 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2024-03-07T03:31:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[79819]: audit 2024-03-07T03:31:23.493462+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.148:0/1706399683' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[79819]: audit 2024-03-07T03:31:23.494192+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[82222]: cluster 2024-03-07T03:31:22.750785+0000 mgr.y (mgr.34101) 523 : cluster 0 pgmap v914: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 326 KiB/s rd, 66 KiB/s wr, 386 op/s 2024-03-07T03:31:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[82222]: cluster 2024-03-07T03:31:23.489544+0000 mon.a (mon.0) 1319 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2024-03-07T03:31:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[82222]: audit 2024-03-07T03:31:23.493462+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.148:0/1706399683' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:24.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:24 smithi148 bash[82222]: audit 2024-03-07T03:31:23.494192+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:24 smithi195 bash[66542]: cluster 2024-03-07T03:31:22.750785+0000 mgr.y (mgr.34101) 523 : cluster 0 pgmap v914: 617 pgs: 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 326 KiB/s rd, 66 KiB/s wr, 386 op/s 2024-03-07T03:31:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:24 smithi195 bash[66542]: cluster 2024-03-07T03:31:23.489544+0000 mon.a (mon.0) 1319 : cluster 0 osdmap e773: 8 total, 8 up, 8 in 2024-03-07T03:31:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:24 smithi195 bash[66542]: audit 2024-03-07T03:31:23.493462+0000 mon.c (mon.2) 78 : audit 1 from='client.? 172.21.15.148:0/1706399683' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:24 smithi195 bash[66542]: audit 2024-03-07T03:31:23.494192+0000 mon.a (mon.0) 1320 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:25.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:25 smithi148 bash[79819]: audit 2024-03-07T03:31:24.484690+0000 mon.a (mon.0) 1321 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:25.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:25 smithi148 bash[79819]: cluster 2024-03-07T03:31:24.492669+0000 mon.a (mon.0) 1322 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2024-03-07T03:31:25.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:25 smithi148 bash[82222]: audit 2024-03-07T03:31:24.484690+0000 mon.a (mon.0) 1321 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:25.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:25 smithi148 bash[82222]: cluster 2024-03-07T03:31:24.492669+0000 mon.a (mon.0) 1322 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2024-03-07T03:31:25.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:25 smithi195 bash[66542]: audit 2024-03-07T03:31:24.484690+0000 mon.a (mon.0) 1321 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:25.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:25 smithi195 bash[66542]: cluster 2024-03-07T03:31:24.492669+0000 mon.a (mon.0) 1322 : cluster 0 osdmap e774: 8 total, 8 up, 8 in 2024-03-07T03:31:26.591 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:26.629 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:26.644 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:26.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:26 smithi148 bash[79819]: cluster 2024-03-07T03:31:24.751950+0000 mgr.y (mgr.34101) 524 : cluster 0 pgmap v917: 649 pgs: 9 creating+peering, 23 unknown, 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 499 KiB/s rd, 34 KiB/s wr, 661 op/s 2024-03-07T03:31:26.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:26 smithi148 bash[79819]: cluster 2024-03-07T03:31:25.520818+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2024-03-07T03:31:26.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:26 smithi148 bash[82222]: cluster 2024-03-07T03:31:24.751950+0000 mgr.y (mgr.34101) 524 : cluster 0 pgmap v917: 649 pgs: 9 creating+peering, 23 unknown, 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 499 KiB/s rd, 34 KiB/s wr, 661 op/s 2024-03-07T03:31:26.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:26 smithi148 bash[82222]: cluster 2024-03-07T03:31:25.520818+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2024-03-07T03:31:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:26 smithi195 bash[66542]: cluster 2024-03-07T03:31:24.751950+0000 mgr.y (mgr.34101) 524 : cluster 0 pgmap v917: 649 pgs: 9 creating+peering, 23 unknown, 617 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 499 KiB/s rd, 34 KiB/s wr, 661 op/s 2024-03-07T03:31:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:26 smithi195 bash[66542]: cluster 2024-03-07T03:31:25.520818+0000 mon.a (mon.0) 1323 : cluster 0 osdmap e775: 8 total, 8 up, 8 in 2024-03-07T03:31:27.522 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:31:27] "GET /metrics HTTP/1.1" 200 75196 "" "Prometheus/2.43.0" 2024-03-07T03:31:27.730 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.PoolStatsPP (5106 ms) 2024-03-07T03:31:27.730 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-03-07T03:31:27.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:27 smithi148 bash[79819]: cluster 2024-03-07T03:31:26.532852+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2024-03-07T03:31:27.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:27 smithi148 bash[79819]: audit 2024-03-07T03:31:27.325839+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:27.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:27 smithi148 bash[82222]: cluster 2024-03-07T03:31:26.532852+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2024-03-07T03:31:27.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:27 smithi148 bash[82222]: audit 2024-03-07T03:31:27.325839+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:27 smithi195 bash[66542]: cluster 2024-03-07T03:31:26.532852+0000 mon.a (mon.0) 1324 : cluster 0 osdmap e776: 8 total, 8 up, 8 in 2024-03-07T03:31:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:27 smithi195 bash[66542]: audit 2024-03-07T03:31:27.325839+0000 mon.a (mon.0) 1325 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:28.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:28 smithi148 bash[79819]: cluster 2024-03-07T03:31:26.753217+0000 mgr.y (mgr.34101) 525 : cluster 0 pgmap v920: 649 pgs: 1 creating+activating, 9 creating+peering, 3 unknown, 636 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 549 KiB/s rd, 36 KiB/s wr, 719 op/s 2024-03-07T03:31:28.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:28 smithi148 bash[79819]: cluster 2024-03-07T03:31:27.543031+0000 mon.a (mon.0) 1326 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2024-03-07T03:31:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:28 smithi148 bash[82222]: cluster 2024-03-07T03:31:26.753217+0000 mgr.y (mgr.34101) 525 : cluster 0 pgmap v920: 649 pgs: 1 creating+activating, 9 creating+peering, 3 unknown, 636 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 549 KiB/s rd, 36 KiB/s wr, 719 op/s 2024-03-07T03:31:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:28 smithi148 bash[82222]: cluster 2024-03-07T03:31:27.543031+0000 mon.a (mon.0) 1326 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2024-03-07T03:31:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:28 smithi195 bash[66542]: cluster 2024-03-07T03:31:26.753217+0000 mgr.y (mgr.34101) 525 : cluster 0 pgmap v920: 649 pgs: 1 creating+activating, 9 creating+peering, 3 unknown, 636 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 549 KiB/s rd, 36 KiB/s wr, 719 op/s 2024-03-07T03:31:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:28 smithi195 bash[66542]: cluster 2024-03-07T03:31:27.543031+0000 mon.a (mon.0) 1326 : cluster 0 osdmap e777: 8 total, 8 up, 8 in 2024-03-07T03:31:29.570 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:29.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:29 smithi195 bash[66542]: cluster 2024-03-07T03:31:28.553917+0000 mon.a (mon.0) 1327 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2024-03-07T03:31:29.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:29 smithi195 bash[66542]: audit 2024-03-07T03:31:28.555611+0000 mon.a (mon.0) 1328 : audit 1 from='client.? 172.21.15.148:0/4188880414' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:30.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:29 smithi148 bash[79819]: cluster 2024-03-07T03:31:28.553917+0000 mon.a (mon.0) 1327 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2024-03-07T03:31:30.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:29 smithi148 bash[79819]: audit 2024-03-07T03:31:28.555611+0000 mon.a (mon.0) 1328 : audit 1 from='client.? 172.21.15.148:0/4188880414' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:30.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:29 smithi148 bash[82222]: cluster 2024-03-07T03:31:28.553917+0000 mon.a (mon.0) 1327 : cluster 0 osdmap e778: 8 total, 8 up, 8 in 2024-03-07T03:31:30.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:29 smithi148 bash[82222]: audit 2024-03-07T03:31:28.555611+0000 mon.a (mon.0) 1328 : audit 1 from='client.? 172.21.15.148:0/4188880414' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-33","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[79819]: cluster 2024-03-07T03:31:28.754625+0000 mgr.y (mgr.34101) 526 : cluster 0 pgmap v923: 681 pgs: 1 creating+activating, 32 unknown, 5 creating+peering, 643 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 176 KiB/s rd, 9.7 KiB/s wr, 219 op/s 2024-03-07T03:31:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[79819]: cluster 2024-03-07T03:31:29.548026+0000 mon.a (mon.0) 1329 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:31:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[79819]: audit 2024-03-07T03:31:29.559784+0000 mon.a (mon.0) 1330 : audit 1 from='client.? 172.21.15.148:0/4188880414' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[79819]: cluster 2024-03-07T03:31:29.574487+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2024-03-07T03:31:31.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[82222]: cluster 2024-03-07T03:31:28.754625+0000 mgr.y (mgr.34101) 526 : cluster 0 pgmap v923: 681 pgs: 1 creating+activating, 32 unknown, 5 creating+peering, 643 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 176 KiB/s rd, 9.7 KiB/s wr, 219 op/s 2024-03-07T03:31:31.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[82222]: cluster 2024-03-07T03:31:29.548026+0000 mon.a (mon.0) 1329 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:31:31.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[82222]: audit 2024-03-07T03:31:29.559784+0000 mon.a (mon.0) 1330 : audit 1 from='client.? 172.21.15.148:0/4188880414' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:31.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:30 smithi148 bash[82222]: cluster 2024-03-07T03:31:29.574487+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2024-03-07T03:31:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:30 smithi195 bash[66542]: cluster 2024-03-07T03:31:28.754625+0000 mgr.y (mgr.34101) 526 : cluster 0 pgmap v923: 681 pgs: 1 creating+activating, 32 unknown, 5 creating+peering, 643 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 176 KiB/s rd, 9.7 KiB/s wr, 219 op/s 2024-03-07T03:31:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:30 smithi195 bash[66542]: cluster 2024-03-07T03:31:29.548026+0000 mon.a (mon.0) 1329 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:31:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:30 smithi195 bash[66542]: audit 2024-03-07T03:31:29.559784+0000 mon.a (mon.0) 1330 : audit 1 from='client.? 172.21.15.148:0/4188880414' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:30 smithi195 bash[66542]: cluster 2024-03-07T03:31:29.574487+0000 mon.a (mon.0) 1331 : cluster 0 osdmap e779: 8 total, 8 up, 8 in 2024-03-07T03:31:32.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:31 smithi148 bash[79819]: cluster 2024-03-07T03:31:30.570138+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2024-03-07T03:31:32.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:31 smithi148 bash[82222]: cluster 2024-03-07T03:31:30.570138+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2024-03-07T03:31:32.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:31 smithi195 bash[66542]: cluster 2024-03-07T03:31:30.570138+0000 mon.a (mon.0) 1332 : cluster 0 osdmap e780: 8 total, 8 up, 8 in 2024-03-07T03:31:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:31:32 smithi195 bash[59134]: ts=2024-03-07T03:31:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:32 smithi148 bash[82222]: cluster 2024-03-07T03:31:30.755863+0000 mgr.y (mgr.34101) 527 : cluster 0 pgmap v926: 681 pgs: 8 creating+peering, 15 unknown, 658 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.0 KiB/s wr, 60 op/s 2024-03-07T03:31:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:32 smithi148 bash[82222]: cluster 2024-03-07T03:31:31.578694+0000 mon.a (mon.0) 1333 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:31:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:32 smithi148 bash[82222]: cluster 2024-03-07T03:31:31.595069+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2024-03-07T03:31:33.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:32 smithi148 bash[79819]: cluster 2024-03-07T03:31:30.755863+0000 mgr.y (mgr.34101) 527 : cluster 0 pgmap v926: 681 pgs: 8 creating+peering, 15 unknown, 658 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.0 KiB/s wr, 60 op/s 2024-03-07T03:31:33.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:32 smithi148 bash[79819]: cluster 2024-03-07T03:31:31.578694+0000 mon.a (mon.0) 1333 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:31:33.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:32 smithi148 bash[79819]: cluster 2024-03-07T03:31:31.595069+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2024-03-07T03:31:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:32 smithi195 bash[66542]: cluster 2024-03-07T03:31:30.755863+0000 mgr.y (mgr.34101) 527 : cluster 0 pgmap v926: 681 pgs: 8 creating+peering, 15 unknown, 658 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.0 KiB/s wr, 60 op/s 2024-03-07T03:31:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:32 smithi195 bash[66542]: cluster 2024-03-07T03:31:31.578694+0000 mon.a (mon.0) 1333 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:31:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:32 smithi195 bash[66542]: cluster 2024-03-07T03:31:31.595069+0000 mon.a (mon.0) 1334 : cluster 0 osdmap e781: 8 total, 8 up, 8 in 2024-03-07T03:31:34.004 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ImageSpec (6274 ms) 2024-03-07T03:31:34.004 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-03-07T03:31:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:33 smithi148 bash[79819]: cluster 2024-03-07T03:31:32.601175+0000 mon.a (mon.0) 1335 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2024-03-07T03:31:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:33 smithi148 bash[82222]: cluster 2024-03-07T03:31:32.601175+0000 mon.a (mon.0) 1335 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2024-03-07T03:31:34.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:33 smithi195 bash[66542]: cluster 2024-03-07T03:31:32.601175+0000 mon.a (mon.0) 1335 : cluster 0 osdmap e782: 8 total, 8 up, 8 in 2024-03-07T03:31:35.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:34 smithi148 bash[79819]: cluster 2024-03-07T03:31:32.756869+0000 mgr.y (mgr.34101) 528 : cluster 0 pgmap v929: 681 pgs: 8 creating+peering, 10 unknown, 663 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.7 KiB/s wr, 47 op/s 2024-03-07T03:31:35.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:34 smithi148 bash[79819]: cluster 2024-03-07T03:31:33.628191+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2024-03-07T03:31:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:34 smithi148 bash[82222]: cluster 2024-03-07T03:31:32.756869+0000 mgr.y (mgr.34101) 528 : cluster 0 pgmap v929: 681 pgs: 8 creating+peering, 10 unknown, 663 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.7 KiB/s wr, 47 op/s 2024-03-07T03:31:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:34 smithi148 bash[82222]: cluster 2024-03-07T03:31:33.628191+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2024-03-07T03:31:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:34 smithi195 bash[66542]: cluster 2024-03-07T03:31:32.756869+0000 mgr.y (mgr.34101) 528 : cluster 0 pgmap v929: 681 pgs: 8 creating+peering, 10 unknown, 663 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 34 KiB/s rd, 4.7 KiB/s wr, 47 op/s 2024-03-07T03:31:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:34 smithi195 bash[66542]: cluster 2024-03-07T03:31:33.628191+0000 mon.a (mon.0) 1336 : cluster 0 osdmap e783: 8 total, 8 up, 8 in 2024-03-07T03:31:35.659 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:36.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:35 smithi148 bash[79819]: cluster 2024-03-07T03:31:34.645313+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2024-03-07T03:31:36.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:35 smithi148 bash[79819]: audit 2024-03-07T03:31:34.647800+0000 mon.a (mon.0) 1338 : audit 1 from='client.? 172.21.15.148:0/1492840830' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:35 smithi148 bash[82222]: cluster 2024-03-07T03:31:34.645313+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2024-03-07T03:31:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:35 smithi148 bash[82222]: audit 2024-03-07T03:31:34.647800+0000 mon.a (mon.0) 1338 : audit 1 from='client.? 172.21.15.148:0/1492840830' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:36.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:35 smithi195 bash[66542]: cluster 2024-03-07T03:31:34.645313+0000 mon.a (mon.0) 1337 : cluster 0 osdmap e784: 8 total, 8 up, 8 in 2024-03-07T03:31:36.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:35 smithi195 bash[66542]: audit 2024-03-07T03:31:34.647800+0000 mon.a (mon.0) 1338 : audit 1 from='client.? 172.21.15.148:0/1492840830' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-34","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[79819]: cluster 2024-03-07T03:31:34.758132+0000 mgr.y (mgr.34101) 529 : cluster 0 pgmap v932: 713 pgs: 32 unknown, 681 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 10 op/s 2024-03-07T03:31:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[79819]: cluster 2024-03-07T03:31:35.632626+0000 mon.a (mon.0) 1339 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:31:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[79819]: audit 2024-03-07T03:31:35.644295+0000 mon.a (mon.0) 1340 : audit 1 from='client.? 172.21.15.148:0/1492840830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[79819]: cluster 2024-03-07T03:31:35.654705+0000 mon.a (mon.0) 1341 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2024-03-07T03:31:37.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[82222]: cluster 2024-03-07T03:31:34.758132+0000 mgr.y (mgr.34101) 529 : cluster 0 pgmap v932: 713 pgs: 32 unknown, 681 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 10 op/s 2024-03-07T03:31:37.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[82222]: cluster 2024-03-07T03:31:35.632626+0000 mon.a (mon.0) 1339 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:31:37.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[82222]: audit 2024-03-07T03:31:35.644295+0000 mon.a (mon.0) 1340 : audit 1 from='client.? 172.21.15.148:0/1492840830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:37.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:36 smithi148 bash[82222]: cluster 2024-03-07T03:31:35.654705+0000 mon.a (mon.0) 1341 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2024-03-07T03:31:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:36 smithi195 bash[66542]: cluster 2024-03-07T03:31:34.758132+0000 mgr.y (mgr.34101) 529 : cluster 0 pgmap v932: 713 pgs: 32 unknown, 681 active+clean; 314 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.5 KiB/s rd, 2.2 KiB/s wr, 10 op/s 2024-03-07T03:31:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:36 smithi195 bash[66542]: cluster 2024-03-07T03:31:35.632626+0000 mon.a (mon.0) 1339 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:31:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:36 smithi195 bash[66542]: audit 2024-03-07T03:31:35.644295+0000 mon.a (mon.0) 1340 : audit 1 from='client.? 172.21.15.148:0/1492840830' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:36 smithi195 bash[66542]: cluster 2024-03-07T03:31:35.654705+0000 mon.a (mon.0) 1341 : cluster 0 osdmap e785: 8 total, 8 up, 8 in 2024-03-07T03:31:37.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:31:37] "GET /metrics HTTP/1.1" 200 77587 "" "Prometheus/2.43.0" 2024-03-07T03:31:38.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:37 smithi148 bash[79819]: cluster 2024-03-07T03:31:36.654099+0000 mon.a (mon.0) 1342 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2024-03-07T03:31:38.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:37 smithi148 bash[79819]: cluster 2024-03-07T03:31:37.357552+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2024-03-07T03:31:38.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:37 smithi148 bash[82222]: cluster 2024-03-07T03:31:36.654099+0000 mon.a (mon.0) 1342 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2024-03-07T03:31:38.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:37 smithi148 bash[82222]: cluster 2024-03-07T03:31:37.357552+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2024-03-07T03:31:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:37 smithi195 bash[66542]: cluster 2024-03-07T03:31:36.654099+0000 mon.a (mon.0) 1342 : cluster 0 osdmap e786: 8 total, 8 up, 8 in 2024-03-07T03:31:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:37 smithi195 bash[66542]: cluster 2024-03-07T03:31:37.357552+0000 mon.a (mon.0) 1343 : cluster 0 osdmap e787: 8 total, 8 up, 8 in 2024-03-07T03:31:39.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[79819]: cluster 2024-03-07T03:31:36.759354+0000 mgr.y (mgr.34101) 530 : cluster 0 pgmap v935: 713 pgs: 8 creating+peering, 23 unknown, 682 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 5.5 KiB/s wr, 150 op/s 2024-03-07T03:31:39.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[79819]: cluster 2024-03-07T03:31:37.657102+0000 mon.a (mon.0) 1344 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:31:39.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[79819]: audit 2024-03-07T03:31:38.254818+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:39.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[79819]: cluster 2024-03-07T03:31:38.361257+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2024-03-07T03:31:39.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[82222]: cluster 2024-03-07T03:31:36.759354+0000 mgr.y (mgr.34101) 530 : cluster 0 pgmap v935: 713 pgs: 8 creating+peering, 23 unknown, 682 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 5.5 KiB/s wr, 150 op/s 2024-03-07T03:31:39.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[82222]: cluster 2024-03-07T03:31:37.657102+0000 mon.a (mon.0) 1344 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:31:39.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[82222]: audit 2024-03-07T03:31:38.254818+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:39.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:38 smithi148 bash[82222]: cluster 2024-03-07T03:31:38.361257+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2024-03-07T03:31:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:38 smithi195 bash[66542]: cluster 2024-03-07T03:31:36.759354+0000 mgr.y (mgr.34101) 530 : cluster 0 pgmap v935: 713 pgs: 8 creating+peering, 23 unknown, 682 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 120 KiB/s rd, 5.5 KiB/s wr, 150 op/s 2024-03-07T03:31:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:38 smithi195 bash[66542]: cluster 2024-03-07T03:31:37.657102+0000 mon.a (mon.0) 1344 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:31:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:38 smithi195 bash[66542]: audit 2024-03-07T03:31:38.254818+0000 mon.a (mon.0) 1345 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:38 smithi195 bash[66542]: cluster 2024-03-07T03:31:38.361257+0000 mon.a (mon.0) 1346 : cluster 0 osdmap e788: 8 total, 8 up, 8 in 2024-03-07T03:31:40.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:40 smithi148 bash[79819]: cluster 2024-03-07T03:31:38.762440+0000 mgr.y (mgr.34101) 531 : cluster 0 pgmap v938: 713 pgs: 8 creating+peering, 10 unknown, 695 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 5.5 KiB/s wr, 167 op/s 2024-03-07T03:31:40.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:40 smithi148 bash[79819]: cluster 2024-03-07T03:31:39.363467+0000 mon.a (mon.0) 1347 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2024-03-07T03:31:40.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:40 smithi148 bash[82222]: cluster 2024-03-07T03:31:38.762440+0000 mgr.y (mgr.34101) 531 : cluster 0 pgmap v938: 713 pgs: 8 creating+peering, 10 unknown, 695 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 5.5 KiB/s wr, 167 op/s 2024-03-07T03:31:40.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:40 smithi148 bash[82222]: cluster 2024-03-07T03:31:39.363467+0000 mon.a (mon.0) 1347 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2024-03-07T03:31:40.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:40 smithi195 bash[66542]: cluster 2024-03-07T03:31:38.762440+0000 mgr.y (mgr.34101) 531 : cluster 0 pgmap v938: 713 pgs: 8 creating+peering, 10 unknown, 695 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 5.5 KiB/s wr, 167 op/s 2024-03-07T03:31:40.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:40 smithi195 bash[66542]: cluster 2024-03-07T03:31:39.363467+0000 mon.a (mon.0) 1347 : cluster 0 osdmap e789: 8 total, 8 up, 8 in 2024-03-07T03:31:42.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:41 smithi148 bash[79819]: cluster 2024-03-07T03:31:40.683680+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2024-03-07T03:31:42.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:41 smithi148 bash[82222]: cluster 2024-03-07T03:31:40.683680+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2024-03-07T03:31:42.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:41 smithi195 bash[66542]: cluster 2024-03-07T03:31:40.683680+0000 mon.a (mon.0) 1348 : cluster 0 osdmap e790: 8 total, 8 up, 8 in 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 3 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 3 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout:child: image218 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout:child: image220 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout:child: image219 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-15/image218/acb852862b2d 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-34/image219/acb876c07a24 2024-03-07T03:31:42.424 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout:found child test-librbd-smithi148-68562-15/image220/acb837e16773 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout:num children is: 0 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout:num snaps is: 0 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout:expected: 0 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8420 ms) 2024-03-07T03:31:42.425 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-03-07T03:31:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:31:42 smithi195 bash[59134]: ts=2024-03-07T03:31:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[79819]: cluster 2024-03-07T03:31:40.763214+0000 mgr.y (mgr.34101) 532 : cluster 0 pgmap v941: 681 pgs: 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[79819]: cluster 2024-03-07T03:31:41.693223+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[79819]: audit 2024-03-07T03:31:42.326338+0000 mon.a (mon.0) 1350 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[79819]: cluster 2024-03-07T03:31:42.361509+0000 mon.a (mon.0) 1351 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[82222]: cluster 2024-03-07T03:31:40.763214+0000 mgr.y (mgr.34101) 532 : cluster 0 pgmap v941: 681 pgs: 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[82222]: cluster 2024-03-07T03:31:41.693223+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[82222]: audit 2024-03-07T03:31:42.326338+0000 mon.a (mon.0) 1350 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:42 smithi148 bash[82222]: cluster 2024-03-07T03:31:42.361509+0000 mon.a (mon.0) 1351 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2024-03-07T03:31:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:42 smithi195 bash[66542]: cluster 2024-03-07T03:31:40.763214+0000 mgr.y (mgr.34101) 532 : cluster 0 pgmap v941: 681 pgs: 681 active+clean; 2.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:31:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:42 smithi195 bash[66542]: cluster 2024-03-07T03:31:41.693223+0000 mon.a (mon.0) 1349 : cluster 0 osdmap e791: 8 total, 8 up, 8 in 2024-03-07T03:31:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:42 smithi195 bash[66542]: audit 2024-03-07T03:31:42.326338+0000 mon.a (mon.0) 1350 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:42 smithi195 bash[66542]: cluster 2024-03-07T03:31:42.361509+0000 mon.a (mon.0) 1351 : cluster 0 osdmap e792: 8 total, 8 up, 8 in 2024-03-07T03:31:44.377 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:44 smithi148 bash[79819]: cluster 2024-03-07T03:31:42.764300+0000 mgr.y (mgr.34101) 533 : cluster 0 pgmap v944: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:31:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:44 smithi148 bash[79819]: cluster 2024-03-07T03:31:43.373048+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2024-03-07T03:31:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:44 smithi148 bash[79819]: audit 2024-03-07T03:31:43.376569+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.148:0/1712131787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:44 smithi148 bash[82222]: cluster 2024-03-07T03:31:42.764300+0000 mgr.y (mgr.34101) 533 : cluster 0 pgmap v944: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:31:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:44 smithi148 bash[82222]: cluster 2024-03-07T03:31:43.373048+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2024-03-07T03:31:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:44 smithi148 bash[82222]: audit 2024-03-07T03:31:43.376569+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.148:0/1712131787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:44 smithi195 bash[66542]: cluster 2024-03-07T03:31:42.764300+0000 mgr.y (mgr.34101) 533 : cluster 0 pgmap v944: 681 pgs: 681 active+clean; 2.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:31:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:44 smithi195 bash[66542]: cluster 2024-03-07T03:31:43.373048+0000 mon.a (mon.0) 1352 : cluster 0 osdmap e793: 8 total, 8 up, 8 in 2024-03-07T03:31:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:44 smithi195 bash[66542]: audit 2024-03-07T03:31:43.376569+0000 mon.a (mon.0) 1353 : audit 1 from='client.? 172.21.15.148:0/1712131787' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-35","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:31:45.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:45 smithi148 bash[79819]: audit 2024-03-07T03:31:44.369057+0000 mon.a (mon.0) 1354 : audit 1 from='client.? 172.21.15.148:0/1712131787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:45.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:45 smithi148 bash[79819]: cluster 2024-03-07T03:31:44.376087+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2024-03-07T03:31:45.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:45 smithi148 bash[82222]: audit 2024-03-07T03:31:44.369057+0000 mon.a (mon.0) 1354 : audit 1 from='client.? 172.21.15.148:0/1712131787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:45.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:45 smithi148 bash[82222]: cluster 2024-03-07T03:31:44.376087+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2024-03-07T03:31:45.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:45 smithi195 bash[66542]: audit 2024-03-07T03:31:44.369057+0000 mon.a (mon.0) 1354 : audit 1 from='client.? 172.21.15.148:0/1712131787' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:31:45.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:45 smithi195 bash[66542]: cluster 2024-03-07T03:31:44.376087+0000 mon.a (mon.0) 1355 : cluster 0 osdmap e794: 8 total, 8 up, 8 in 2024-03-07T03:31:46.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:46 smithi148 bash[79819]: cluster 2024-03-07T03:31:44.765491+0000 mgr.y (mgr.34101) 534 : cluster 0 pgmap v947: 713 pgs: 22 creating+peering, 10 unknown, 681 active+clean; 2.6 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 8.0 KiB/s wr, 165 op/s 2024-03-07T03:31:46.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:46 smithi148 bash[79819]: cluster 2024-03-07T03:31:45.406219+0000 mon.a (mon.0) 1356 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2024-03-07T03:31:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:46 smithi148 bash[82222]: cluster 2024-03-07T03:31:44.765491+0000 mgr.y (mgr.34101) 534 : cluster 0 pgmap v947: 713 pgs: 22 creating+peering, 10 unknown, 681 active+clean; 2.6 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 8.0 KiB/s wr, 165 op/s 2024-03-07T03:31:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:46 smithi148 bash[82222]: cluster 2024-03-07T03:31:45.406219+0000 mon.a (mon.0) 1356 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2024-03-07T03:31:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:46 smithi195 bash[66542]: cluster 2024-03-07T03:31:44.765491+0000 mgr.y (mgr.34101) 534 : cluster 0 pgmap v947: 713 pgs: 22 creating+peering, 10 unknown, 681 active+clean; 2.6 MiB data, 789 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 8.0 KiB/s wr, 165 op/s 2024-03-07T03:31:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:46 smithi195 bash[66542]: cluster 2024-03-07T03:31:45.406219+0000 mon.a (mon.0) 1356 : cluster 0 osdmap e795: 8 total, 8 up, 8 in 2024-03-07T03:31:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:31:47] "GET /metrics HTTP/1.1" 200 79970 "" "Prometheus/2.43.0" 2024-03-07T03:31:47.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:47 smithi148 bash[79819]: cluster 2024-03-07T03:31:46.414193+0000 mon.a (mon.0) 1357 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2024-03-07T03:31:47.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:47 smithi148 bash[79819]: cluster 2024-03-07T03:31:47.365037+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2024-03-07T03:31:47.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:47 smithi148 bash[82222]: cluster 2024-03-07T03:31:46.414193+0000 mon.a (mon.0) 1357 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2024-03-07T03:31:47.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:47 smithi148 bash[82222]: cluster 2024-03-07T03:31:47.365037+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2024-03-07T03:31:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:47 smithi195 bash[66542]: cluster 2024-03-07T03:31:46.414193+0000 mon.a (mon.0) 1357 : cluster 0 osdmap e796: 8 total, 8 up, 8 in 2024-03-07T03:31:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:47 smithi195 bash[66542]: cluster 2024-03-07T03:31:47.365037+0000 mon.a (mon.0) 1358 : cluster 0 osdmap e797: 8 total, 8 up, 8 in 2024-03-07T03:31:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:48 smithi148 bash[79819]: cluster 2024-03-07T03:31:46.766729+0000 mgr.y (mgr.34101) 535 : cluster 0 pgmap v950: 713 pgs: 22 creating+peering, 691 active+clean; 2.6 MiB data, 490 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 8.0 KiB/s wr, 165 op/s 2024-03-07T03:31:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:48 smithi148 bash[79819]: audit 2024-03-07T03:31:48.341903+0000 mon.a (mon.0) 1359 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:48.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:48 smithi148 bash[82222]: cluster 2024-03-07T03:31:46.766729+0000 mgr.y (mgr.34101) 535 : cluster 0 pgmap v950: 713 pgs: 22 creating+peering, 691 active+clean; 2.6 MiB data, 490 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 8.0 KiB/s wr, 165 op/s 2024-03-07T03:31:48.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:48 smithi148 bash[82222]: audit 2024-03-07T03:31:48.341903+0000 mon.a (mon.0) 1359 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:48 smithi195 bash[66542]: cluster 2024-03-07T03:31:46.766729+0000 mgr.y (mgr.34101) 535 : cluster 0 pgmap v950: 713 pgs: 22 creating+peering, 691 active+clean; 2.6 MiB data, 490 MiB used, 715 GiB / 715 GiB avail; 129 KiB/s rd, 8.0 KiB/s wr, 165 op/s 2024-03-07T03:31:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:48 smithi195 bash[66542]: audit 2024-03-07T03:31:48.341903+0000 mon.a (mon.0) 1359 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:49.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:49 smithi148 bash[79819]: cluster 2024-03-07T03:31:48.434304+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2024-03-07T03:31:49.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:49 smithi148 bash[82222]: cluster 2024-03-07T03:31:48.434304+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2024-03-07T03:31:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:49 smithi195 bash[66542]: cluster 2024-03-07T03:31:48.434304+0000 mon.a (mon.0) 1360 : cluster 0 osdmap e798: 8 total, 8 up, 8 in 2024-03-07T03:31:50.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:50 smithi148 bash[79819]: cluster 2024-03-07T03:31:48.770857+0000 mgr.y (mgr.34101) 536 : cluster 0 pgmap v953: 713 pgs: 12 creating+peering, 701 active+clean; 2.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 2 op/s 2024-03-07T03:31:50.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:50 smithi148 bash[79819]: cluster 2024-03-07T03:31:49.452909+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2024-03-07T03:31:50.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:50 smithi148 bash[82222]: cluster 2024-03-07T03:31:48.770857+0000 mgr.y (mgr.34101) 536 : cluster 0 pgmap v953: 713 pgs: 12 creating+peering, 701 active+clean; 2.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 2 op/s 2024-03-07T03:31:50.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:50 smithi148 bash[82222]: cluster 2024-03-07T03:31:49.452909+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2024-03-07T03:31:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:50 smithi195 bash[66542]: cluster 2024-03-07T03:31:48.770857+0000 mgr.y (mgr.34101) 536 : cluster 0 pgmap v953: 713 pgs: 12 creating+peering, 701 active+clean; 2.6 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 511 B/s wr, 2 op/s 2024-03-07T03:31:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:50 smithi195 bash[66542]: cluster 2024-03-07T03:31:49.452909+0000 mon.a (mon.0) 1361 : cluster 0 osdmap e799: 8 total, 8 up, 8 in 2024-03-07T03:31:51.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:51 smithi148 bash[79819]: cluster 2024-03-07T03:31:50.448855+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2024-03-07T03:31:51.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:51 smithi148 bash[82222]: cluster 2024-03-07T03:31:50.448855+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2024-03-07T03:31:51.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:51 smithi195 bash[66542]: cluster 2024-03-07T03:31:50.448855+0000 mon.a (mon.0) 1362 : cluster 0 osdmap e800: 8 total, 8 up, 8 in 2024-03-07T03:31:52.530 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.QuiesceWatch (10106 ms) 2024-03-07T03:31:52.530 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-03-07T03:31:52.530 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:31:52 smithi195 bash[59134]: ts=2024-03-07T03:31:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:31:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:52 smithi195 bash[66542]: cluster 2024-03-07T03:31:50.772488+0000 mgr.y (mgr.34101) 537 : cluster 0 pgmap v956: 713 pgs: 713 active+clean; 2.6 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.0 KiB/s wr, 58 op/s 2024-03-07T03:31:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:52 smithi195 bash[66542]: cluster 2024-03-07T03:31:51.477688+0000 mon.a (mon.0) 1363 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2024-03-07T03:31:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:52 smithi195 bash[66542]: cluster 2024-03-07T03:31:52.370945+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2024-03-07T03:31:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:52 smithi148 bash[79819]: cluster 2024-03-07T03:31:50.772488+0000 mgr.y (mgr.34101) 537 : cluster 0 pgmap v956: 713 pgs: 713 active+clean; 2.6 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.0 KiB/s wr, 58 op/s 2024-03-07T03:31:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:52 smithi148 bash[79819]: cluster 2024-03-07T03:31:51.477688+0000 mon.a (mon.0) 1363 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2024-03-07T03:31:52.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:52 smithi148 bash[79819]: cluster 2024-03-07T03:31:52.370945+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2024-03-07T03:31:52.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:52 smithi148 bash[82222]: cluster 2024-03-07T03:31:50.772488+0000 mgr.y (mgr.34101) 537 : cluster 0 pgmap v956: 713 pgs: 713 active+clean; 2.6 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.0 KiB/s wr, 58 op/s 2024-03-07T03:31:52.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:52 smithi148 bash[82222]: cluster 2024-03-07T03:31:51.477688+0000 mon.a (mon.0) 1363 : cluster 0 osdmap e801: 8 total, 8 up, 8 in 2024-03-07T03:31:52.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:52 smithi148 bash[82222]: cluster 2024-03-07T03:31:52.370945+0000 mon.a (mon.0) 1364 : cluster 0 osdmap e802: 8 total, 8 up, 8 in 2024-03-07T03:31:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:54 smithi148 bash[79819]: cluster 2024-03-07T03:31:52.773686+0000 mgr.y (mgr.34101) 538 : cluster 0 pgmap v959: 713 pgs: 713 active+clean; 2.6 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.7 KiB/s wr, 58 op/s 2024-03-07T03:31:54.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:54 smithi148 bash[79819]: cluster 2024-03-07T03:31:53.484578+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2024-03-07T03:31:54.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:54 smithi148 bash[82222]: cluster 2024-03-07T03:31:52.773686+0000 mgr.y (mgr.34101) 538 : cluster 0 pgmap v959: 713 pgs: 713 active+clean; 2.6 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.7 KiB/s wr, 58 op/s 2024-03-07T03:31:54.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:54 smithi148 bash[82222]: cluster 2024-03-07T03:31:53.484578+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2024-03-07T03:31:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:54 smithi195 bash[66542]: cluster 2024-03-07T03:31:52.773686+0000 mgr.y (mgr.34101) 538 : cluster 0 pgmap v959: 713 pgs: 713 active+clean; 2.6 MiB data, 479 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 3.7 KiB/s wr, 58 op/s 2024-03-07T03:31:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:54 smithi195 bash[66542]: cluster 2024-03-07T03:31:53.484578+0000 mon.a (mon.0) 1365 : cluster 0 osdmap e803: 8 total, 8 up, 8 in 2024-03-07T03:31:55.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:55 smithi148 bash[79819]: cluster 2024-03-07T03:31:54.487629+0000 mon.a (mon.0) 1366 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2024-03-07T03:31:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:55 smithi148 bash[82222]: cluster 2024-03-07T03:31:54.487629+0000 mon.a (mon.0) 1366 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2024-03-07T03:31:55.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:55 smithi195 bash[66542]: cluster 2024-03-07T03:31:54.487629+0000 mon.a (mon.0) 1366 : cluster 0 osdmap e804: 8 total, 8 up, 8 in 2024-03-07T03:31:56.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:56 smithi148 bash[79819]: cluster 2024-03-07T03:31:54.775026+0000 mgr.y (mgr.34101) 539 : cluster 0 pgmap v962: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 2.0 KiB/s wr, 25 op/s 2024-03-07T03:31:56.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:56 smithi148 bash[79819]: cluster 2024-03-07T03:31:55.526958+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2024-03-07T03:31:56.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:56 smithi148 bash[82222]: cluster 2024-03-07T03:31:54.775026+0000 mgr.y (mgr.34101) 539 : cluster 0 pgmap v962: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 2.0 KiB/s wr, 25 op/s 2024-03-07T03:31:56.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:56 smithi148 bash[82222]: cluster 2024-03-07T03:31:55.526958+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2024-03-07T03:31:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:56 smithi195 bash[66542]: cluster 2024-03-07T03:31:54.775026+0000 mgr.y (mgr.34101) 539 : cluster 0 pgmap v962: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 2.0 KiB/s wr, 25 op/s 2024-03-07T03:31:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:56 smithi195 bash[66542]: cluster 2024-03-07T03:31:55.526958+0000 mon.a (mon.0) 1367 : cluster 0 osdmap e805: 8 total, 8 up, 8 in 2024-03-07T03:31:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:31:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:31:57] "GET /metrics HTTP/1.1" 200 79970 "" "Prometheus/2.43.0" 2024-03-07T03:31:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:57 smithi195 bash[66542]: cluster 2024-03-07T03:31:56.531712+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2024-03-07T03:31:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:57 smithi195 bash[66542]: audit 2024-03-07T03:31:57.326632+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:57 smithi195 bash[66542]: cluster 2024-03-07T03:31:57.370585+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2024-03-07T03:31:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:57 smithi148 bash[79819]: cluster 2024-03-07T03:31:56.531712+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2024-03-07T03:31:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:57 smithi148 bash[79819]: audit 2024-03-07T03:31:57.326632+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:57 smithi148 bash[79819]: cluster 2024-03-07T03:31:57.370585+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2024-03-07T03:31:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:57 smithi148 bash[82222]: cluster 2024-03-07T03:31:56.531712+0000 mon.a (mon.0) 1368 : cluster 0 osdmap e806: 8 total, 8 up, 8 in 2024-03-07T03:31:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:57 smithi148 bash[82222]: audit 2024-03-07T03:31:57.326632+0000 mon.a (mon.0) 1369 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:31:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:57 smithi148 bash[82222]: cluster 2024-03-07T03:31:57.370585+0000 mon.a (mon.0) 1370 : cluster 0 osdmap e807: 8 total, 8 up, 8 in 2024-03-07T03:31:58.624 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (6094 ms) 2024-03-07T03:31:58.624 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-03-07T03:31:58.624 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:31:58.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:58 smithi195 bash[66542]: cluster 2024-03-07T03:31:56.776591+0000 mgr.y (mgr.34101) 540 : cluster 0 pgmap v965: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-03-07T03:31:58.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:58 smithi195 bash[66542]: cluster 2024-03-07T03:31:58.374977+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2024-03-07T03:31:58.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:31:58 smithi195 bash[66542]: audit 2024-03-07T03:31:58.422415+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:58 smithi148 bash[82222]: cluster 2024-03-07T03:31:56.776591+0000 mgr.y (mgr.34101) 540 : cluster 0 pgmap v965: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-03-07T03:31:59.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:58 smithi148 bash[82222]: cluster 2024-03-07T03:31:58.374977+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2024-03-07T03:31:59.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:31:58 smithi148 bash[82222]: audit 2024-03-07T03:31:58.422415+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:31:59.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:58 smithi148 bash[79819]: cluster 2024-03-07T03:31:56.776591+0000 mgr.y (mgr.34101) 540 : cluster 0 pgmap v965: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 4.5 KiB/s wr, 80 op/s 2024-03-07T03:31:59.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:58 smithi148 bash[79819]: cluster 2024-03-07T03:31:58.374977+0000 mon.a (mon.0) 1371 : cluster 0 osdmap e808: 8 total, 8 up, 8 in 2024-03-07T03:31:59.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:31:58 smithi148 bash[79819]: audit 2024-03-07T03:31:58.422415+0000 mon.a (mon.0) 1372 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:01.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:00 smithi148 bash[79819]: cluster 2024-03-07T03:31:58.780130+0000 mgr.y (mgr.34101) 541 : cluster 0 pgmap v968: 713 pgs: 713 active+clean; 2.6 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 8.2 KiB/s wr, 127 op/s 2024-03-07T03:32:01.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:00 smithi148 bash[79819]: cluster 2024-03-07T03:31:59.572614+0000 mon.a (mon.0) 1373 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2024-03-07T03:32:01.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:00 smithi148 bash[82222]: cluster 2024-03-07T03:31:58.780130+0000 mgr.y (mgr.34101) 541 : cluster 0 pgmap v968: 713 pgs: 713 active+clean; 2.6 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 8.2 KiB/s wr, 127 op/s 2024-03-07T03:32:01.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:00 smithi148 bash[82222]: cluster 2024-03-07T03:31:59.572614+0000 mon.a (mon.0) 1373 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2024-03-07T03:32:01.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:00 smithi195 bash[66542]: cluster 2024-03-07T03:31:58.780130+0000 mgr.y (mgr.34101) 541 : cluster 0 pgmap v968: 713 pgs: 713 active+clean; 2.6 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 94 KiB/s rd, 8.2 KiB/s wr, 127 op/s 2024-03-07T03:32:01.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:00 smithi195 bash[66542]: cluster 2024-03-07T03:31:59.572614+0000 mon.a (mon.0) 1373 : cluster 0 osdmap e809: 8 total, 8 up, 8 in 2024-03-07T03:32:02.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:01 smithi148 bash[79819]: cluster 2024-03-07T03:32:00.574641+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2024-03-07T03:32:02.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:01 smithi148 bash[82222]: cluster 2024-03-07T03:32:00.574641+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2024-03-07T03:32:02.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:01 smithi195 bash[66542]: cluster 2024-03-07T03:32:00.574641+0000 mon.a (mon.0) 1374 : cluster 0 osdmap e810: 8 total, 8 up, 8 in 2024-03-07T03:32:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:32:02 smithi195 bash[59134]: ts=2024-03-07T03:32:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:03.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:02 smithi148 bash[82222]: cluster 2024-03-07T03:32:00.781361+0000 mgr.y (mgr.34101) 542 : cluster 0 pgmap v971: 713 pgs: 713 active+clean; 2.6 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 6.2 KiB/s wr, 77 op/s 2024-03-07T03:32:03.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:02 smithi148 bash[82222]: cluster 2024-03-07T03:32:01.602434+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2024-03-07T03:32:03.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:02 smithi148 bash[82222]: cluster 2024-03-07T03:32:02.374196+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2024-03-07T03:32:03.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:02 smithi148 bash[79819]: cluster 2024-03-07T03:32:00.781361+0000 mgr.y (mgr.34101) 542 : cluster 0 pgmap v971: 713 pgs: 713 active+clean; 2.6 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 6.2 KiB/s wr, 77 op/s 2024-03-07T03:32:03.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:02 smithi148 bash[79819]: cluster 2024-03-07T03:32:01.602434+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2024-03-07T03:32:03.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:02 smithi148 bash[79819]: cluster 2024-03-07T03:32:02.374196+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2024-03-07T03:32:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:02 smithi195 bash[66542]: cluster 2024-03-07T03:32:00.781361+0000 mgr.y (mgr.34101) 542 : cluster 0 pgmap v971: 713 pgs: 713 active+clean; 2.6 MiB data, 486 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 6.2 KiB/s wr, 77 op/s 2024-03-07T03:32:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:02 smithi195 bash[66542]: cluster 2024-03-07T03:32:01.602434+0000 mon.a (mon.0) 1375 : cluster 0 osdmap e811: 8 total, 8 up, 8 in 2024-03-07T03:32:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:02 smithi195 bash[66542]: cluster 2024-03-07T03:32:02.374196+0000 mon.a (mon.0) 1376 : cluster 0 osdmap e812: 8 total, 8 up, 8 in 2024-03-07T03:32:04.581 INFO:tasks.workunit.client.0.smithi148.stdout:10.00% done 2024-03-07T03:32:04.581 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.QuiesceWatchError (5957 ms) 2024-03-07T03:32:04.581 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-03-07T03:32:04.581 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:32:04.612 INFO:tasks.workunit.client.0.smithi148.stderr:test quiesce is not long enough to time out 2024-03-07T03:32:04.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:04 smithi148 bash[79819]: cluster 2024-03-07T03:32:02.782564+0000 mgr.y (mgr.34101) 543 : cluster 0 pgmap v974: 713 pgs: 713 active+clean; 2.6 MiB data, 490 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.2 KiB/s wr, 47 op/s 2024-03-07T03:32:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:04 smithi148 bash[79819]: cluster 2024-03-07T03:32:03.378833+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2024-03-07T03:32:04.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:04 smithi148 bash[82222]: cluster 2024-03-07T03:32:02.782564+0000 mgr.y (mgr.34101) 543 : cluster 0 pgmap v974: 713 pgs: 713 active+clean; 2.6 MiB data, 490 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.2 KiB/s wr, 47 op/s 2024-03-07T03:32:04.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:04 smithi148 bash[82222]: cluster 2024-03-07T03:32:03.378833+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2024-03-07T03:32:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:04 smithi195 bash[66542]: cluster 2024-03-07T03:32:02.782564+0000 mgr.y (mgr.34101) 543 : cluster 0 pgmap v974: 713 pgs: 713 active+clean; 2.6 MiB data, 490 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 2.2 KiB/s wr, 47 op/s 2024-03-07T03:32:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:04 smithi195 bash[66542]: cluster 2024-03-07T03:32:03.378833+0000 mon.a (mon.0) 1377 : cluster 0 osdmap e813: 8 total, 8 up, 8 in 2024-03-07T03:32:05.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:05 smithi148 bash[79819]: cluster 2024-03-07T03:32:04.382429+0000 mon.a (mon.0) 1378 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2024-03-07T03:32:05.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:05 smithi148 bash[82222]: cluster 2024-03-07T03:32:04.382429+0000 mon.a (mon.0) 1378 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2024-03-07T03:32:05.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:05 smithi195 bash[66542]: cluster 2024-03-07T03:32:04.382429+0000 mon.a (mon.0) 1378 : cluster 0 osdmap e814: 8 total, 8 up, 8 in 2024-03-07T03:32:06.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:06 smithi148 bash[79819]: cluster 2024-03-07T03:32:04.783898+0000 mgr.y (mgr.34101) 544 : cluster 0 pgmap v977: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.0 KiB/s wr, 77 op/s 2024-03-07T03:32:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:06 smithi148 bash[82222]: cluster 2024-03-07T03:32:04.783898+0000 mgr.y (mgr.34101) 544 : cluster 0 pgmap v977: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.0 KiB/s wr, 77 op/s 2024-03-07T03:32:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:06 smithi195 bash[66542]: cluster 2024-03-07T03:32:04.783898+0000 mgr.y (mgr.34101) 544 : cluster 0 pgmap v977: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.0 KiB/s wr, 77 op/s 2024-03-07T03:32:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:32:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:32:07] "GET /metrics HTTP/1.1" 200 82375 "" "Prometheus/2.43.0" 2024-03-07T03:32:08.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:08 smithi148 bash[79819]: cluster 2024-03-07T03:32:06.787129+0000 mgr.y (mgr.34101) 545 : cluster 0 pgmap v978: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 3.9 KiB/s wr, 83 op/s 2024-03-07T03:32:08.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:08 smithi148 bash[79819]: cluster 2024-03-07T03:32:07.375697+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2024-03-07T03:32:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:08 smithi148 bash[82222]: cluster 2024-03-07T03:32:06.787129+0000 mgr.y (mgr.34101) 545 : cluster 0 pgmap v978: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 3.9 KiB/s wr, 83 op/s 2024-03-07T03:32:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:08 smithi148 bash[82222]: cluster 2024-03-07T03:32:07.375697+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2024-03-07T03:32:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:08 smithi195 bash[66542]: cluster 2024-03-07T03:32:06.787129+0000 mgr.y (mgr.34101) 545 : cluster 0 pgmap v978: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 3.9 KiB/s wr, 83 op/s 2024-03-07T03:32:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:08 smithi195 bash[66542]: cluster 2024-03-07T03:32:07.375697+0000 mon.a (mon.0) 1379 : cluster 0 osdmap e815: 8 total, 8 up, 8 in 2024-03-07T03:32:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:10 smithi148 bash[79819]: cluster 2024-03-07T03:32:08.790634+0000 mgr.y (mgr.34101) 546 : cluster 0 pgmap v980: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 4.0 KiB/s wr, 75 op/s 2024-03-07T03:32:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:10 smithi148 bash[82222]: cluster 2024-03-07T03:32:08.790634+0000 mgr.y (mgr.34101) 546 : cluster 0 pgmap v980: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 4.0 KiB/s wr, 75 op/s 2024-03-07T03:32:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:10 smithi195 bash[66542]: cluster 2024-03-07T03:32:08.790634+0000 mgr.y (mgr.34101) 546 : cluster 0 pgmap v980: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 4.0 KiB/s wr, 75 op/s 2024-03-07T03:32:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:12 smithi195 bash[66542]: cluster 2024-03-07T03:32:10.793976+0000 mgr.y (mgr.34101) 547 : cluster 0 pgmap v981: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 3.2 KiB/s wr, 64 op/s 2024-03-07T03:32:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:12 smithi195 bash[66542]: audit 2024-03-07T03:32:12.326984+0000 mon.a (mon.0) 1380 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:12.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:32:12 smithi195 bash[59134]: ts=2024-03-07T03:32:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:12 smithi148 bash[79819]: cluster 2024-03-07T03:32:10.793976+0000 mgr.y (mgr.34101) 547 : cluster 0 pgmap v981: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 3.2 KiB/s wr, 64 op/s 2024-03-07T03:32:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:12 smithi148 bash[79819]: audit 2024-03-07T03:32:12.326984+0000 mon.a (mon.0) 1380 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:12 smithi148 bash[82222]: cluster 2024-03-07T03:32:10.793976+0000 mgr.y (mgr.34101) 547 : cluster 0 pgmap v981: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 3.2 KiB/s wr, 64 op/s 2024-03-07T03:32:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:12 smithi148 bash[82222]: audit 2024-03-07T03:32:12.326984+0000 mon.a (mon.0) 1380 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:13.463 INFO:tasks.workunit.client.0.smithi148.stderr:test quiesce is timed out 2024-03-07T03:32:13.467 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 0 2024-03-07T03:32:13.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:13 smithi148 bash[79819]: cluster 2024-03-07T03:32:12.381078+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2024-03-07T03:32:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:13 smithi148 bash[82222]: cluster 2024-03-07T03:32:12.381078+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2024-03-07T03:32:13.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:13 smithi195 bash[66542]: cluster 2024-03-07T03:32:12.381078+0000 mon.a (mon.0) 1381 : cluster 0 osdmap e816: 8 total, 8 up, 8 in 2024-03-07T03:32:14.467 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 1 2024-03-07T03:32:14.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:14 smithi148 bash[79819]: cluster 2024-03-07T03:32:12.795186+0000 mgr.y (mgr.34101) 548 : cluster 0 pgmap v983: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 KiB/s wr, 41 op/s 2024-03-07T03:32:14.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:14 smithi148 bash[79819]: cluster 2024-03-07T03:32:13.402595+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2024-03-07T03:32:14.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:14 smithi148 bash[82222]: cluster 2024-03-07T03:32:12.795186+0000 mgr.y (mgr.34101) 548 : cluster 0 pgmap v983: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 KiB/s wr, 41 op/s 2024-03-07T03:32:14.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:14 smithi148 bash[82222]: cluster 2024-03-07T03:32:13.402595+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2024-03-07T03:32:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:14 smithi195 bash[66542]: cluster 2024-03-07T03:32:12.795186+0000 mgr.y (mgr.34101) 548 : cluster 0 pgmap v983: 713 pgs: 713 active+clean; 2.6 MiB data, 495 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 KiB/s wr, 41 op/s 2024-03-07T03:32:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:14 smithi195 bash[66542]: cluster 2024-03-07T03:32:13.402595+0000 mon.a (mon.0) 1382 : cluster 0 osdmap e817: 8 total, 8 up, 8 in 2024-03-07T03:32:15.468 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 2 2024-03-07T03:32:16.468 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 3 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: cluster 2024-03-07T03:32:14.798457+0000 mgr.y (mgr.34101) 549 : cluster 0 pgmap v985: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 551 B/s wr, 5 op/s 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.818421+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]: dispatch 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.818950+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1", "id": [0, 6]}]: dispatch 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.819405+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [4, 3]}]: dispatch 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.819864+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.15", "id": [0, 3, 4, 6]}]: dispatch 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.820362+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]: dispatch 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.820793+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-03-07T03:32:16.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.821181+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.f", "id": [0, 1]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.821636+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[79819]: audit 2024-03-07T03:32:15.822042+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: cluster 2024-03-07T03:32:14.798457+0000 mgr.y (mgr.34101) 549 : cluster 0 pgmap v985: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 551 B/s wr, 5 op/s 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.818421+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.818950+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1", "id": [0, 6]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.819405+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [4, 3]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.819864+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.15", "id": [0, 3, 4, 6]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.820362+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]: dispatch 2024-03-07T03:32:16.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.820793+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-03-07T03:32:16.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.821181+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.f", "id": [0, 1]}]: dispatch 2024-03-07T03:32:16.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.821636+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-03-07T03:32:16.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:16 smithi148 bash[82222]: audit 2024-03-07T03:32:15.822042+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]: dispatch 2024-03-07T03:32:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: cluster 2024-03-07T03:32:14.798457+0000 mgr.y (mgr.34101) 549 : cluster 0 pgmap v985: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 551 B/s wr, 5 op/s 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.818421+0000 mon.a (mon.0) 1383 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.818950+0000 mon.a (mon.0) 1384 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1", "id": [0, 6]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.819405+0000 mon.a (mon.0) 1385 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [4, 3]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.819864+0000 mon.a (mon.0) 1386 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.15", "id": [0, 3, 4, 6]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.820362+0000 mon.a (mon.0) 1387 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.820793+0000 mon.a (mon.0) 1388 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.821181+0000 mon.a (mon.0) 1389 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.f", "id": [0, 1]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.821636+0000 mon.a (mon.0) 1390 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]: dispatch 2024-03-07T03:32:16.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:16 smithi195 bash[66542]: audit 2024-03-07T03:32:15.822042+0000 mon.a (mon.0) 1391 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]: dispatch 2024-03-07T03:32:17.424 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:32:17] "GET /metrics HTTP/1.1" 200 82379 "" "Prometheus/2.43.0" 2024-03-07T03:32:17.468 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 4 2024-03-07T03:32:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.405091+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]': finished 2024-03-07T03:32:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.405488+0000 mon.a (mon.0) 1393 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1", "id": [0, 6]}]': finished 2024-03-07T03:32:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.405717+0000 mon.a (mon.0) 1394 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [4, 3]}]': finished 2024-03-07T03:32:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.406008+0000 mon.a (mon.0) 1395 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.15", "id": [0, 3, 4, 6]}]': finished 2024-03-07T03:32:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.406239+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]': finished 2024-03-07T03:32:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.406421+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-03-07T03:32:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.406678+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.f", "id": [0, 1]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.406925+0000 mon.a (mon.0) 1399 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: audit 2024-03-07T03:32:16.407164+0000 mon.a (mon.0) 1400 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[82222]: cluster 2024-03-07T03:32:16.415968+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.405091+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.405488+0000 mon.a (mon.0) 1393 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1", "id": [0, 6]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.405717+0000 mon.a (mon.0) 1394 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [4, 3]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.406008+0000 mon.a (mon.0) 1395 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.15", "id": [0, 3, 4, 6]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.406239+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]': finished 2024-03-07T03:32:17.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.406421+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-03-07T03:32:17.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.406678+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.f", "id": [0, 1]}]': finished 2024-03-07T03:32:17.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.406925+0000 mon.a (mon.0) 1399 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-03-07T03:32:17.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: audit 2024-03-07T03:32:16.407164+0000 mon.a (mon.0) 1400 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-03-07T03:32:17.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:17 smithi148 bash[79819]: cluster 2024-03-07T03:32:16.415968+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2024-03-07T03:32:17.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.405091+0000 mon.a (mon.0) 1392 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]': finished 2024-03-07T03:32:17.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.405488+0000 mon.a (mon.0) 1393 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1", "id": [0, 6]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.405717+0000 mon.a (mon.0) 1394 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.4", "id": [4, 3]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.406008+0000 mon.a (mon.0) 1395 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.15", "id": [0, 3, 4, 6]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.406239+0000 mon.a (mon.0) 1396 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.6", "id": [0, 1]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.406421+0000 mon.a (mon.0) 1397 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.a", "id": [3, 1]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.406678+0000 mon.a (mon.0) 1398 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.f", "id": [0, 1]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.406925+0000 mon.a (mon.0) 1399 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.17", "id": [3, 1]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: audit 2024-03-07T03:32:16.407164+0000 mon.a (mon.0) 1400 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.1e", "id": [3, 7]}]': finished 2024-03-07T03:32:17.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:17 smithi195 bash[66542]: cluster 2024-03-07T03:32:16.415968+0000 mon.a (mon.0) 1401 : cluster 0 osdmap e818: 8 total, 8 up, 8 in 2024-03-07T03:32:18.468 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 5 2024-03-07T03:32:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:18 smithi148 bash[79819]: cluster 2024-03-07T03:32:16.799736+0000 mgr.y (mgr.34101) 550 : cluster 0 pgmap v987: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-03-07T03:32:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:18 smithi148 bash[79819]: cluster 2024-03-07T03:32:17.446549+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2024-03-07T03:32:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:18 smithi148 bash[82222]: cluster 2024-03-07T03:32:16.799736+0000 mgr.y (mgr.34101) 550 : cluster 0 pgmap v987: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-03-07T03:32:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:18 smithi148 bash[82222]: cluster 2024-03-07T03:32:17.446549+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2024-03-07T03:32:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:18 smithi195 bash[66542]: cluster 2024-03-07T03:32:16.799736+0000 mgr.y (mgr.34101) 550 : cluster 0 pgmap v987: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 4.7 KiB/s rd, 511 B/s wr, 5 op/s 2024-03-07T03:32:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:18 smithi195 bash[66542]: cluster 2024-03-07T03:32:17.446549+0000 mon.a (mon.0) 1402 : cluster 0 osdmap e819: 8 total, 8 up, 8 in 2024-03-07T03:32:19.468 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 6 2024-03-07T03:32:20.468 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 7 2024-03-07T03:32:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:20 smithi148 bash[79819]: cluster 2024-03-07T03:32:18.803320+0000 mgr.y (mgr.34101) 551 : cluster 0 pgmap v989: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 511 B/s wr, 6 op/s; 8 B/s, 0 objects/s recovering 2024-03-07T03:32:20.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:20 smithi148 bash[82222]: cluster 2024-03-07T03:32:18.803320+0000 mgr.y (mgr.34101) 551 : cluster 0 pgmap v989: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 511 B/s wr, 6 op/s; 8 B/s, 0 objects/s recovering 2024-03-07T03:32:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:20 smithi195 bash[66542]: cluster 2024-03-07T03:32:18.803320+0000 mgr.y (mgr.34101) 551 : cluster 0 pgmap v989: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 511 B/s wr, 6 op/s; 8 B/s, 0 objects/s recovering 2024-03-07T03:32:21.469 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 8 2024-03-07T03:32:22.469 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 9 2024-03-07T03:32:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:22 smithi195 bash[66542]: cluster 2024-03-07T03:32:20.806729+0000 mgr.y (mgr.34101) 552 : cluster 0 pgmap v990: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 414 B/s wr, 4 op/s; 11 B/s, 0 objects/s recovering 2024-03-07T03:32:22.567 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:32:22 smithi195 bash[59134]: ts=2024-03-07T03:32:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:22.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:22 smithi148 bash[79819]: cluster 2024-03-07T03:32:20.806729+0000 mgr.y (mgr.34101) 552 : cluster 0 pgmap v990: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 414 B/s wr, 4 op/s; 11 B/s, 0 objects/s recovering 2024-03-07T03:32:22.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:22 smithi148 bash[82222]: cluster 2024-03-07T03:32:20.806729+0000 mgr.y (mgr.34101) 552 : cluster 0 pgmap v990: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.5 KiB/s rd, 414 B/s wr, 4 op/s; 11 B/s, 0 objects/s recovering 2024-03-07T03:32:23.469 INFO:tasks.workunit.client.0.smithi148.stderr:waiting for timed out ... 10 2024-03-07T03:32:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:23 smithi148 bash[79819]: audit 2024-03-07T03:32:22.532417+0000 mon.a (mon.0) 1403 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:32:23.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:23 smithi148 bash[79819]: audit 2024-03-07T03:32:23.061650+0000 mon.a (mon.0) 1404 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:23.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:23 smithi148 bash[79819]: audit 2024-03-07T03:32:23.067849+0000 mon.a (mon.0) 1405 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:23.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:23 smithi148 bash[82222]: audit 2024-03-07T03:32:22.532417+0000 mon.a (mon.0) 1403 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:32:23.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:23 smithi148 bash[82222]: audit 2024-03-07T03:32:23.061650+0000 mon.a (mon.0) 1404 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:23.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:23 smithi148 bash[82222]: audit 2024-03-07T03:32:23.067849+0000 mon.a (mon.0) 1405 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:23.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:23 smithi195 bash[66542]: audit 2024-03-07T03:32:22.532417+0000 mon.a (mon.0) 1403 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:32:23.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:23 smithi195 bash[66542]: audit 2024-03-07T03:32:23.061650+0000 mon.a (mon.0) 1404 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:23.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:23 smithi195 bash[66542]: audit 2024-03-07T03:32:23.067849+0000 mon.a (mon.0) 1405 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:24 smithi148 bash[79819]: cluster 2024-03-07T03:32:22.808695+0000 mgr.y (mgr.34101) 553 : cluster 0 pgmap v991: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 3 op/s; 15 B/s, 0 objects/s recovering 2024-03-07T03:32:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:24 smithi148 bash[82222]: cluster 2024-03-07T03:32:22.808695+0000 mgr.y (mgr.34101) 553 : cluster 0 pgmap v991: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 3 op/s; 15 B/s, 0 objects/s recovering 2024-03-07T03:32:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:24 smithi195 bash[66542]: cluster 2024-03-07T03:32:22.808695+0000 mgr.y (mgr.34101) 553 : cluster 0 pgmap v991: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 383 B/s wr, 3 op/s; 15 B/s, 0 objects/s recovering 2024-03-07T03:32:26.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:26 smithi148 bash[79819]: cluster 2024-03-07T03:32:24.812969+0000 mgr.y (mgr.34101) 554 : cluster 0 pgmap v992: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 14 B/s, 0 objects/s recovering 2024-03-07T03:32:26.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:26 smithi148 bash[82222]: cluster 2024-03-07T03:32:24.812969+0000 mgr.y (mgr.34101) 554 : cluster 0 pgmap v992: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 14 B/s, 0 objects/s recovering 2024-03-07T03:32:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:26 smithi195 bash[66542]: cluster 2024-03-07T03:32:24.812969+0000 mgr.y (mgr.34101) 554 : cluster 0 pgmap v992: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 14 B/s, 0 objects/s recovering 2024-03-07T03:32:27.462 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:32:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:32:27] "GET /metrics HTTP/1.1" 200 82379 "" "Prometheus/2.43.0" 2024-03-07T03:32:27.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:27 smithi148 bash[79819]: audit 2024-03-07T03:32:27.327389+0000 mon.a (mon.0) 1406 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:27.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:27 smithi148 bash[82222]: audit 2024-03-07T03:32:27.327389+0000 mon.a (mon.0) 1406 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:27 smithi195 bash[66542]: audit 2024-03-07T03:32:27.327389+0000 mon.a (mon.0) 1406 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:28.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:28 smithi148 bash[79819]: cluster 2024-03-07T03:32:26.816231+0000 mgr.y (mgr.34101) 555 : cluster 0 pgmap v993: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 12 B/s, 0 objects/s recovering 2024-03-07T03:32:28.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:28 smithi148 bash[79819]: audit 2024-03-07T03:32:28.400657+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:28.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:28 smithi148 bash[79819]: audit 2024-03-07T03:32:28.409348+0000 mon.a (mon.0) 1408 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:28 smithi148 bash[82222]: cluster 2024-03-07T03:32:26.816231+0000 mgr.y (mgr.34101) 555 : cluster 0 pgmap v993: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 12 B/s, 0 objects/s recovering 2024-03-07T03:32:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:28 smithi148 bash[82222]: audit 2024-03-07T03:32:28.400657+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:28 smithi148 bash[82222]: audit 2024-03-07T03:32:28.409348+0000 mon.a (mon.0) 1408 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:28 smithi195 bash[66542]: cluster 2024-03-07T03:32:26.816231+0000 mgr.y (mgr.34101) 555 : cluster 0 pgmap v993: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 12 B/s, 0 objects/s recovering 2024-03-07T03:32:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:28 smithi195 bash[66542]: audit 2024-03-07T03:32:28.400657+0000 mon.a (mon.0) 1407 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:28 smithi195 bash[66542]: audit 2024-03-07T03:32:28.409348+0000 mon.a (mon.0) 1408 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[79819]: cluster 2024-03-07T03:32:28.818195+0000 mgr.y (mgr.34101) 556 : cluster 0 pgmap v994: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 10 B/s, 0 objects/s recovering 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[79819]: audit 2024-03-07T03:32:29.010592+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[79819]: audit 2024-03-07T03:32:29.018481+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[79819]: audit 2024-03-07T03:32:29.646834+0000 mon.a (mon.0) 1411 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[79819]: audit 2024-03-07T03:32:29.648842+0000 mon.a (mon.0) 1412 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[79819]: audit 2024-03-07T03:32:29.659369+0000 mon.a (mon.0) 1413 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[82222]: cluster 2024-03-07T03:32:28.818195+0000 mgr.y (mgr.34101) 556 : cluster 0 pgmap v994: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 10 B/s, 0 objects/s recovering 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[82222]: audit 2024-03-07T03:32:29.010592+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[82222]: audit 2024-03-07T03:32:29.018481+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[82222]: audit 2024-03-07T03:32:29.646834+0000 mon.a (mon.0) 1411 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:32:30.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[82222]: audit 2024-03-07T03:32:29.648842+0000 mon.a (mon.0) 1412 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:32:30.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:30 smithi148 bash[82222]: audit 2024-03-07T03:32:29.659369+0000 mon.a (mon.0) 1413 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:30 smithi195 bash[66542]: cluster 2024-03-07T03:32:28.818195+0000 mgr.y (mgr.34101) 556 : cluster 0 pgmap v994: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 10 B/s, 0 objects/s recovering 2024-03-07T03:32:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:30 smithi195 bash[66542]: audit 2024-03-07T03:32:29.010592+0000 mon.a (mon.0) 1409 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:30 smithi195 bash[66542]: audit 2024-03-07T03:32:29.018481+0000 mon.a (mon.0) 1410 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:30 smithi195 bash[66542]: audit 2024-03-07T03:32:29.646834+0000 mon.a (mon.0) 1411 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:32:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:30 smithi195 bash[66542]: audit 2024-03-07T03:32:29.648842+0000 mon.a (mon.0) 1412 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:32:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:30 smithi195 bash[66542]: audit 2024-03-07T03:32:29.659369+0000 mon.a (mon.0) 1413 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:32:32.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:32 smithi148 bash[79819]: cluster 2024-03-07T03:32:30.821429+0000 mgr.y (mgr.34101) 557 : cluster 0 pgmap v995: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-03-07T03:32:32.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:32 smithi148 bash[82222]: cluster 2024-03-07T03:32:30.821429+0000 mgr.y (mgr.34101) 557 : cluster 0 pgmap v995: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-03-07T03:32:32.305 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:32 smithi195 bash[66542]: cluster 2024-03-07T03:32:30.821429+0000 mgr.y (mgr.34101) 557 : cluster 0 pgmap v995: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-03-07T03:32:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:32:32 smithi195 bash[59134]: ts=2024-03-07T03:32:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:34.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:34 smithi148 bash[79819]: cluster 2024-03-07T03:32:32.823505+0000 mgr.y (mgr.34101) 558 : cluster 0 pgmap v996: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-03-07T03:32:34.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:34 smithi148 bash[82222]: cluster 2024-03-07T03:32:32.823505+0000 mgr.y (mgr.34101) 558 : cluster 0 pgmap v996: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-03-07T03:32:34.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:34 smithi195 bash[66542]: cluster 2024-03-07T03:32:32.823505+0000 mgr.y (mgr.34101) 558 : cluster 0 pgmap v996: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 2 B/s, 0 objects/s recovering 2024-03-07T03:32:34.469 INFO:tasks.workunit.client.0.smithi148.stderr:test retry succeeds 2024-03-07T03:32:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:36 smithi148 bash[79819]: cluster 2024-03-07T03:32:34.827883+0000 mgr.y (mgr.34101) 559 : cluster 0 pgmap v997: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:32:36.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:36 smithi148 bash[79819]: cluster 2024-03-07T03:32:35.039186+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2024-03-07T03:32:36.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:36 smithi148 bash[82222]: cluster 2024-03-07T03:32:34.827883+0000 mgr.y (mgr.34101) 559 : cluster 0 pgmap v997: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:32:36.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:36 smithi148 bash[82222]: cluster 2024-03-07T03:32:35.039186+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2024-03-07T03:32:36.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:36 smithi195 bash[66542]: cluster 2024-03-07T03:32:34.827883+0000 mgr.y (mgr.34101) 559 : cluster 0 pgmap v997: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:32:36.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:36 smithi195 bash[66542]: cluster 2024-03-07T03:32:35.039186+0000 mon.a (mon.0) 1414 : cluster 0 osdmap e820: 8 total, 8 up, 8 in 2024-03-07T03:32:37.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32708 ms) 2024-03-07T03:32:37.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-03-07T03:32:37.288 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:32:37.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:37 smithi195 bash[66542]: cluster 2024-03-07T03:32:36.040790+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2024-03-07T03:32:37.430 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.WriteZeroes (142 ms) 2024-03-07T03:32:37.430 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-03-07T03:32:37.430 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:32:37.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:37 smithi148 bash[79819]: cluster 2024-03-07T03:32:36.040790+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2024-03-07T03:32:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:32:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:32:37] "GET /metrics HTTP/1.1" 200 82375 "" "Prometheus/2.43.0" 2024-03-07T03:32:37.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:37 smithi148 bash[82222]: cluster 2024-03-07T03:32:36.040790+0000 mon.a (mon.0) 1415 : cluster 0 osdmap e821: 8 total, 8 up, 8 in 2024-03-07T03:32:37.577 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (146 ms) 2024-03-07T03:32:37.577 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-03-07T03:32:37.577 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:32:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:38 smithi148 bash[79819]: cluster 2024-03-07T03:32:36.829978+0000 mgr.y (mgr.34101) 560 : cluster 0 pgmap v1000: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 766 B/s rd, 0 B/s wr, 1 op/s 2024-03-07T03:32:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:38 smithi148 bash[79819]: cluster 2024-03-07T03:32:37.070122+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2024-03-07T03:32:38.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:38 smithi148 bash[82222]: cluster 2024-03-07T03:32:36.829978+0000 mgr.y (mgr.34101) 560 : cluster 0 pgmap v1000: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 766 B/s rd, 0 B/s wr, 1 op/s 2024-03-07T03:32:38.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:38 smithi148 bash[82222]: cluster 2024-03-07T03:32:37.070122+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2024-03-07T03:32:38.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:38 smithi195 bash[66542]: cluster 2024-03-07T03:32:36.829978+0000 mgr.y (mgr.34101) 560 : cluster 0 pgmap v1000: 713 pgs: 713 active+clean; 2.6 MiB data, 496 MiB used, 715 GiB / 715 GiB avail; 766 B/s rd, 0 B/s wr, 1 op/s 2024-03-07T03:32:38.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:38 smithi195 bash[66542]: cluster 2024-03-07T03:32:37.070122+0000 mon.a (mon.0) 1416 : cluster 0 osdmap e822: 8 total, 8 up, 8 in 2024-03-07T03:32:39.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:39 smithi148 bash[79819]: cluster 2024-03-07T03:32:38.068892+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2024-03-07T03:32:39.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:39 smithi148 bash[82222]: cluster 2024-03-07T03:32:38.068892+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2024-03-07T03:32:39.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:39 smithi195 bash[66542]: cluster 2024-03-07T03:32:38.068892+0000 mon.a (mon.0) 1417 : cluster 0 osdmap e823: 8 total, 8 up, 8 in 2024-03-07T03:32:40.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:40 smithi148 bash[79819]: cluster 2024-03-07T03:32:38.833575+0000 mgr.y (mgr.34101) 561 : cluster 0 pgmap v1003: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 2.7 KiB/s wr, 89 op/s 2024-03-07T03:32:40.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:40 smithi148 bash[79819]: cluster 2024-03-07T03:32:39.094462+0000 mon.a (mon.0) 1418 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2024-03-07T03:32:40.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:40 smithi148 bash[82222]: cluster 2024-03-07T03:32:38.833575+0000 mgr.y (mgr.34101) 561 : cluster 0 pgmap v1003: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 2.7 KiB/s wr, 89 op/s 2024-03-07T03:32:40.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:40 smithi148 bash[82222]: cluster 2024-03-07T03:32:39.094462+0000 mon.a (mon.0) 1418 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2024-03-07T03:32:40.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:40 smithi195 bash[66542]: cluster 2024-03-07T03:32:38.833575+0000 mgr.y (mgr.34101) 561 : cluster 0 pgmap v1003: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 2.7 KiB/s wr, 89 op/s 2024-03-07T03:32:40.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:40 smithi195 bash[66542]: cluster 2024-03-07T03:32:39.094462+0000 mon.a (mon.0) 1418 : cluster 0 osdmap e824: 8 total, 8 up, 8 in 2024-03-07T03:32:41.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:41 smithi148 bash[82222]: cluster 2024-03-07T03:32:40.104004+0000 mon.a (mon.0) 1419 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2024-03-07T03:32:41.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:41 smithi148 bash[79819]: cluster 2024-03-07T03:32:40.104004+0000 mon.a (mon.0) 1419 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2024-03-07T03:32:41.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:41 smithi195 bash[66542]: cluster 2024-03-07T03:32:40.104004+0000 mon.a (mon.0) 1419 : cluster 0 osdmap e825: 8 total, 8 up, 8 in 2024-03-07T03:32:42.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:42 smithi148 bash[79819]: cluster 2024-03-07T03:32:40.835573+0000 mgr.y (mgr.34101) 562 : cluster 0 pgmap v1006: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 9.2 KiB/s wr, 161 op/s 2024-03-07T03:32:42.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:42 smithi148 bash[79819]: cluster 2024-03-07T03:32:41.114634+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2024-03-07T03:32:42.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:42 smithi148 bash[82222]: cluster 2024-03-07T03:32:40.835573+0000 mgr.y (mgr.34101) 562 : cluster 0 pgmap v1006: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 9.2 KiB/s wr, 161 op/s 2024-03-07T03:32:42.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:42 smithi148 bash[82222]: cluster 2024-03-07T03:32:41.114634+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2024-03-07T03:32:42.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:42 smithi195 bash[66542]: cluster 2024-03-07T03:32:40.835573+0000 mgr.y (mgr.34101) 562 : cluster 0 pgmap v1006: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 115 KiB/s rd, 9.2 KiB/s wr, 161 op/s 2024-03-07T03:32:42.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:42 smithi195 bash[66542]: cluster 2024-03-07T03:32:41.114634+0000 mon.a (mon.0) 1420 : cluster 0 osdmap e826: 8 total, 8 up, 8 in 2024-03-07T03:32:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:32:42 smithi195 bash[59134]: ts=2024-03-07T03:32:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:43.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:43 smithi148 bash[79819]: cluster 2024-03-07T03:32:42.117001+0000 mon.a (mon.0) 1421 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2024-03-07T03:32:43.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:43 smithi148 bash[79819]: audit 2024-03-07T03:32:42.327858+0000 mon.a (mon.0) 1422 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:43.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:43 smithi148 bash[79819]: cluster 2024-03-07T03:32:42.391386+0000 mon.a (mon.0) 1423 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2024-03-07T03:32:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:43 smithi148 bash[82222]: cluster 2024-03-07T03:32:42.117001+0000 mon.a (mon.0) 1421 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2024-03-07T03:32:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:43 smithi148 bash[82222]: audit 2024-03-07T03:32:42.327858+0000 mon.a (mon.0) 1422 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:43.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:43 smithi148 bash[82222]: cluster 2024-03-07T03:32:42.391386+0000 mon.a (mon.0) 1423 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2024-03-07T03:32:43.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:43 smithi195 bash[66542]: cluster 2024-03-07T03:32:42.117001+0000 mon.a (mon.0) 1421 : cluster 0 osdmap e827: 8 total, 8 up, 8 in 2024-03-07T03:32:43.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:43 smithi195 bash[66542]: audit 2024-03-07T03:32:42.327858+0000 mon.a (mon.0) 1422 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:43.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:43 smithi195 bash[66542]: cluster 2024-03-07T03:32:42.391386+0000 mon.a (mon.0) 1423 : cluster 0 osdmap e828: 8 total, 8 up, 8 in 2024-03-07T03:32:44.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:44 smithi148 bash[79819]: cluster 2024-03-07T03:32:42.836822+0000 mgr.y (mgr.34101) 563 : cluster 0 pgmap v1010: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 8.0 KiB/s wr, 97 op/s 2024-03-07T03:32:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:44 smithi148 bash[79819]: cluster 2024-03-07T03:32:43.396318+0000 mon.a (mon.0) 1424 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2024-03-07T03:32:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:44 smithi148 bash[82222]: cluster 2024-03-07T03:32:42.836822+0000 mgr.y (mgr.34101) 563 : cluster 0 pgmap v1010: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 8.0 KiB/s wr, 97 op/s 2024-03-07T03:32:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:44 smithi148 bash[82222]: cluster 2024-03-07T03:32:43.396318+0000 mon.a (mon.0) 1424 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2024-03-07T03:32:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:44 smithi195 bash[66542]: cluster 2024-03-07T03:32:42.836822+0000 mgr.y (mgr.34101) 563 : cluster 0 pgmap v1010: 713 pgs: 713 active+clean; 2.6 MiB data, 497 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 8.0 KiB/s wr, 97 op/s 2024-03-07T03:32:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:44 smithi195 bash[66542]: cluster 2024-03-07T03:32:43.396318+0000 mon.a (mon.0) 1424 : cluster 0 osdmap e829: 8 total, 8 up, 8 in 2024-03-07T03:32:45.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:45 smithi148 bash[79819]: cluster 2024-03-07T03:32:44.413134+0000 mon.a (mon.0) 1425 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2024-03-07T03:32:45.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:45 smithi148 bash[82222]: cluster 2024-03-07T03:32:44.413134+0000 mon.a (mon.0) 1425 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2024-03-07T03:32:45.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:45 smithi195 bash[66542]: cluster 2024-03-07T03:32:44.413134+0000 mon.a (mon.0) 1425 : cluster 0 osdmap e830: 8 total, 8 up, 8 in 2024-03-07T03:32:46.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:46 smithi148 bash[79819]: cluster 2024-03-07T03:32:44.838088+0000 mgr.y (mgr.34101) 564 : cluster 0 pgmap v1013: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 5.1 KiB/s wr, 37 op/s 2024-03-07T03:32:46.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:46 smithi148 bash[79819]: cluster 2024-03-07T03:32:45.425730+0000 mon.a (mon.0) 1426 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2024-03-07T03:32:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:46 smithi148 bash[82222]: cluster 2024-03-07T03:32:44.838088+0000 mgr.y (mgr.34101) 564 : cluster 0 pgmap v1013: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 5.1 KiB/s wr, 37 op/s 2024-03-07T03:32:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:46 smithi148 bash[82222]: cluster 2024-03-07T03:32:45.425730+0000 mon.a (mon.0) 1426 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2024-03-07T03:32:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:46 smithi195 bash[66542]: cluster 2024-03-07T03:32:44.838088+0000 mgr.y (mgr.34101) 564 : cluster 0 pgmap v1013: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 25 KiB/s rd, 5.1 KiB/s wr, 37 op/s 2024-03-07T03:32:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:46 smithi195 bash[66542]: cluster 2024-03-07T03:32:45.425730+0000 mon.a (mon.0) 1426 : cluster 0 osdmap e831: 8 total, 8 up, 8 in 2024-03-07T03:32:47.423 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:32:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:32:47] "GET /metrics HTTP/1.1" 200 82380 "" "Prometheus/2.43.0" 2024-03-07T03:32:47.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:47 smithi148 bash[82222]: cluster 2024-03-07T03:32:46.430901+0000 mon.a (mon.0) 1427 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2024-03-07T03:32:47.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:47 smithi148 bash[82222]: cluster 2024-03-07T03:32:47.390321+0000 mon.a (mon.0) 1428 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2024-03-07T03:32:47.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:47 smithi148 bash[79819]: cluster 2024-03-07T03:32:46.430901+0000 mon.a (mon.0) 1427 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2024-03-07T03:32:47.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:47 smithi148 bash[79819]: cluster 2024-03-07T03:32:47.390321+0000 mon.a (mon.0) 1428 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2024-03-07T03:32:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:47 smithi195 bash[66542]: cluster 2024-03-07T03:32:46.430901+0000 mon.a (mon.0) 1427 : cluster 0 osdmap e832: 8 total, 8 up, 8 in 2024-03-07T03:32:47.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:47 smithi195 bash[66542]: cluster 2024-03-07T03:32:47.390321+0000 mon.a (mon.0) 1428 : cluster 0 osdmap e833: 8 total, 8 up, 8 in 2024-03-07T03:32:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:48 smithi148 bash[79819]: cluster 2024-03-07T03:32:46.839288+0000 mgr.y (mgr.34101) 565 : cluster 0 pgmap v1016: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 KiB/s wr, 17 op/s 2024-03-07T03:32:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:48 smithi148 bash[82222]: cluster 2024-03-07T03:32:46.839288+0000 mgr.y (mgr.34101) 565 : cluster 0 pgmap v1016: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 KiB/s wr, 17 op/s 2024-03-07T03:32:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:48 smithi195 bash[66542]: cluster 2024-03-07T03:32:46.839288+0000 mgr.y (mgr.34101) 565 : cluster 0 pgmap v1016: 713 pgs: 713 active+clean; 2.6 MiB data, 498 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 4.0 KiB/s wr, 17 op/s 2024-03-07T03:32:49.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:49 smithi148 bash[82222]: cluster 2024-03-07T03:32:48.458209+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2024-03-07T03:32:49.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:49 smithi148 bash[79819]: cluster 2024-03-07T03:32:48.458209+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2024-03-07T03:32:49.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:49 smithi195 bash[66542]: cluster 2024-03-07T03:32:48.458209+0000 mon.a (mon.0) 1429 : cluster 0 osdmap e834: 8 total, 8 up, 8 in 2024-03-07T03:32:50.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:50 smithi148 bash[79819]: cluster 2024-03-07T03:32:48.841828+0000 mgr.y (mgr.34101) 566 : cluster 0 pgmap v1019: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 177 KiB/s rd, 4.5 KiB/s wr, 240 op/s 2024-03-07T03:32:50.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:50 smithi148 bash[79819]: cluster 2024-03-07T03:32:49.466906+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2024-03-07T03:32:50.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:50 smithi148 bash[82222]: cluster 2024-03-07T03:32:48.841828+0000 mgr.y (mgr.34101) 566 : cluster 0 pgmap v1019: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 177 KiB/s rd, 4.5 KiB/s wr, 240 op/s 2024-03-07T03:32:50.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:50 smithi148 bash[82222]: cluster 2024-03-07T03:32:49.466906+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2024-03-07T03:32:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:50 smithi195 bash[66542]: cluster 2024-03-07T03:32:48.841828+0000 mgr.y (mgr.34101) 566 : cluster 0 pgmap v1019: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 177 KiB/s rd, 4.5 KiB/s wr, 240 op/s 2024-03-07T03:32:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:50 smithi195 bash[66542]: cluster 2024-03-07T03:32:49.466906+0000 mon.a (mon.0) 1430 : cluster 0 osdmap e835: 8 total, 8 up, 8 in 2024-03-07T03:32:51.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:51 smithi148 bash[79819]: cluster 2024-03-07T03:32:50.469119+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2024-03-07T03:32:51.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:51 smithi148 bash[82222]: cluster 2024-03-07T03:32:50.469119+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2024-03-07T03:32:51.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:51 smithi195 bash[66542]: cluster 2024-03-07T03:32:50.469119+0000 mon.a (mon.0) 1431 : cluster 0 osdmap e836: 8 total, 8 up, 8 in 2024-03-07T03:32:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:52 smithi195 bash[66542]: cluster 2024-03-07T03:32:50.843035+0000 mgr.y (mgr.34101) 567 : cluster 0 pgmap v1022: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 4.2 KiB/s wr, 242 op/s 2024-03-07T03:32:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:52 smithi195 bash[66542]: cluster 2024-03-07T03:32:51.494385+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2024-03-07T03:32:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:32:52 smithi195 bash[59134]: ts=2024-03-07T03:32:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:32:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:52 smithi148 bash[79819]: cluster 2024-03-07T03:32:50.843035+0000 mgr.y (mgr.34101) 567 : cluster 0 pgmap v1022: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 4.2 KiB/s wr, 242 op/s 2024-03-07T03:32:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:52 smithi148 bash[79819]: cluster 2024-03-07T03:32:51.494385+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2024-03-07T03:32:52.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:52 smithi148 bash[82222]: cluster 2024-03-07T03:32:50.843035+0000 mgr.y (mgr.34101) 567 : cluster 0 pgmap v1022: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 178 KiB/s rd, 4.2 KiB/s wr, 242 op/s 2024-03-07T03:32:52.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:52 smithi148 bash[82222]: cluster 2024-03-07T03:32:51.494385+0000 mon.a (mon.0) 1432 : cluster 0 osdmap e837: 8 total, 8 up, 8 in 2024-03-07T03:32:53.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:53 smithi148 bash[79819]: cluster 2024-03-07T03:32:52.493855+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2024-03-07T03:32:53.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:53 smithi148 bash[82222]: cluster 2024-03-07T03:32:52.493855+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2024-03-07T03:32:53.817 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:53 smithi195 bash[66542]: cluster 2024-03-07T03:32:52.493855+0000 mon.a (mon.0) 1433 : cluster 0 osdmap e838: 8 total, 8 up, 8 in 2024-03-07T03:32:54.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:54 smithi148 bash[79819]: cluster 2024-03-07T03:32:52.844223+0000 mgr.y (mgr.34101) 568 : cluster 0 pgmap v1025: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:32:54.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:54 smithi148 bash[79819]: cluster 2024-03-07T03:32:53.525942+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2024-03-07T03:32:54.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:54 smithi148 bash[82222]: cluster 2024-03-07T03:32:52.844223+0000 mgr.y (mgr.34101) 568 : cluster 0 pgmap v1025: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:32:54.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:54 smithi148 bash[82222]: cluster 2024-03-07T03:32:53.525942+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2024-03-07T03:32:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:54 smithi195 bash[66542]: cluster 2024-03-07T03:32:52.844223+0000 mgr.y (mgr.34101) 568 : cluster 0 pgmap v1025: 713 pgs: 713 active+clean; 2.6 MiB data, 499 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 255 B/s wr, 3 op/s 2024-03-07T03:32:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:54 smithi195 bash[66542]: cluster 2024-03-07T03:32:53.525942+0000 mon.a (mon.0) 1434 : cluster 0 osdmap e839: 8 total, 8 up, 8 in 2024-03-07T03:32:55.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:55 smithi148 bash[79819]: cluster 2024-03-07T03:32:54.523900+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2024-03-07T03:32:55.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:55 smithi148 bash[82222]: cluster 2024-03-07T03:32:54.523900+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2024-03-07T03:32:55.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:55 smithi195 bash[66542]: cluster 2024-03-07T03:32:54.523900+0000 mon.a (mon.0) 1435 : cluster 0 osdmap e840: 8 total, 8 up, 8 in 2024-03-07T03:32:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:56 smithi148 bash[79819]: cluster 2024-03-07T03:32:54.845404+0000 mgr.y (mgr.34101) 569 : cluster 0 pgmap v1028: 713 pgs: 713 active+clean; 2.6 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 23 op/s 2024-03-07T03:32:56.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:56 smithi148 bash[79819]: cluster 2024-03-07T03:32:55.555375+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2024-03-07T03:32:56.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:56 smithi148 bash[82222]: cluster 2024-03-07T03:32:54.845404+0000 mgr.y (mgr.34101) 569 : cluster 0 pgmap v1028: 713 pgs: 713 active+clean; 2.6 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 23 op/s 2024-03-07T03:32:56.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:56 smithi148 bash[82222]: cluster 2024-03-07T03:32:55.555375+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2024-03-07T03:32:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:56 smithi195 bash[66542]: cluster 2024-03-07T03:32:54.845404+0000 mgr.y (mgr.34101) 569 : cluster 0 pgmap v1028: 713 pgs: 713 active+clean; 2.6 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 23 op/s 2024-03-07T03:32:56.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:56 smithi195 bash[66542]: cluster 2024-03-07T03:32:55.555375+0000 mon.a (mon.0) 1436 : cluster 0 osdmap e841: 8 total, 8 up, 8 in 2024-03-07T03:32:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:32:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:32:57] "GET /metrics HTTP/1.1" 200 82380 "" "Prometheus/2.43.0" 2024-03-07T03:32:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:57 smithi195 bash[66542]: cluster 2024-03-07T03:32:56.562386+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2024-03-07T03:32:57.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:57 smithi195 bash[66542]: audit 2024-03-07T03:32:57.328173+0000 mon.a (mon.0) 1438 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:57.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:57 smithi195 bash[66542]: cluster 2024-03-07T03:32:57.396259+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2024-03-07T03:32:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:57 smithi148 bash[79819]: cluster 2024-03-07T03:32:56.562386+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2024-03-07T03:32:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:57 smithi148 bash[79819]: audit 2024-03-07T03:32:57.328173+0000 mon.a (mon.0) 1438 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:57 smithi148 bash[79819]: cluster 2024-03-07T03:32:57.396259+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2024-03-07T03:32:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:57 smithi148 bash[82222]: cluster 2024-03-07T03:32:56.562386+0000 mon.a (mon.0) 1437 : cluster 0 osdmap e842: 8 total, 8 up, 8 in 2024-03-07T03:32:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:57 smithi148 bash[82222]: audit 2024-03-07T03:32:57.328173+0000 mon.a (mon.0) 1438 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:32:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:57 smithi148 bash[82222]: cluster 2024-03-07T03:32:57.396259+0000 mon.a (mon.0) 1439 : cluster 0 osdmap e843: 8 total, 8 up, 8 in 2024-03-07T03:32:58.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:58 smithi195 bash[66542]: cluster 2024-03-07T03:32:56.846637+0000 mgr.y (mgr.34101) 570 : cluster 0 pgmap v1031: 713 pgs: 713 active+clean; 2.6 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 4.7 KiB/s wr, 23 op/s 2024-03-07T03:32:58.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:32:58 smithi195 bash[66542]: cluster 2024-03-07T03:32:58.401107+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2024-03-07T03:32:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:58 smithi148 bash[79819]: cluster 2024-03-07T03:32:56.846637+0000 mgr.y (mgr.34101) 570 : cluster 0 pgmap v1031: 713 pgs: 713 active+clean; 2.6 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 4.7 KiB/s wr, 23 op/s 2024-03-07T03:32:59.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:32:58 smithi148 bash[79819]: cluster 2024-03-07T03:32:58.401107+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2024-03-07T03:32:59.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:58 smithi148 bash[82222]: cluster 2024-03-07T03:32:56.846637+0000 mgr.y (mgr.34101) 570 : cluster 0 pgmap v1031: 713 pgs: 713 active+clean; 2.6 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 4.7 KiB/s wr, 23 op/s 2024-03-07T03:32:59.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:32:58 smithi148 bash[82222]: cluster 2024-03-07T03:32:58.401107+0000 mon.a (mon.0) 1440 : cluster 0 osdmap e844: 8 total, 8 up, 8 in 2024-03-07T03:33:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:00 smithi148 bash[79819]: cluster 2024-03-07T03:32:58.849165+0000 mgr.y (mgr.34101) 571 : cluster 0 pgmap v1034: 713 pgs: 713 active+clean; 2.6 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-07T03:33:00.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:00 smithi148 bash[79819]: cluster 2024-03-07T03:32:59.400074+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2024-03-07T03:33:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:00 smithi148 bash[82222]: cluster 2024-03-07T03:32:58.849165+0000 mgr.y (mgr.34101) 571 : cluster 0 pgmap v1034: 713 pgs: 713 active+clean; 2.6 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-07T03:33:00.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:00 smithi148 bash[82222]: cluster 2024-03-07T03:32:59.400074+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2024-03-07T03:33:00.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:00 smithi195 bash[66542]: cluster 2024-03-07T03:32:58.849165+0000 mgr.y (mgr.34101) 571 : cluster 0 pgmap v1034: 713 pgs: 713 active+clean; 2.6 MiB data, 505 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 52 op/s 2024-03-07T03:33:00.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:00 smithi195 bash[66542]: cluster 2024-03-07T03:32:59.400074+0000 mon.a (mon.0) 1441 : cluster 0 osdmap e845: 8 total, 8 up, 8 in 2024-03-07T03:33:01.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:01 smithi148 bash[79819]: cluster 2024-03-07T03:33:00.412256+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2024-03-07T03:33:01.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:01 smithi148 bash[82222]: cluster 2024-03-07T03:33:00.412256+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2024-03-07T03:33:01.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:01 smithi195 bash[66542]: cluster 2024-03-07T03:33:00.412256+0000 mon.a (mon.0) 1442 : cluster 0 osdmap e846: 8 total, 8 up, 8 in 2024-03-07T03:33:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:02 smithi195 bash[66542]: cluster 2024-03-07T03:33:00.851265+0000 mgr.y (mgr.34101) 572 : cluster 0 pgmap v1037: 713 pgs: 713 active+clean; 2.6 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 53 op/s 2024-03-07T03:33:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:02 smithi195 bash[66542]: cluster 2024-03-07T03:33:01.427854+0000 mon.a (mon.0) 1443 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2024-03-07T03:33:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:02 smithi195 bash[66542]: cluster 2024-03-07T03:33:02.397761+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2024-03-07T03:33:02.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:33:02 smithi195 bash[59134]: ts=2024-03-07T03:33:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:02.579 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] TestLibRBD.ConcurentOperations (25003 ms) 2024-03-07T03:33:02.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:02 smithi148 bash[82222]: cluster 2024-03-07T03:33:00.851265+0000 mgr.y (mgr.34101) 572 : cluster 0 pgmap v1037: 713 pgs: 713 active+clean; 2.6 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 53 op/s 2024-03-07T03:33:02.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:02 smithi148 bash[82222]: cluster 2024-03-07T03:33:01.427854+0000 mon.a (mon.0) 1443 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2024-03-07T03:33:02.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:02 smithi148 bash[82222]: cluster 2024-03-07T03:33:02.397761+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2024-03-07T03:33:02.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:02 smithi148 bash[79819]: cluster 2024-03-07T03:33:00.851265+0000 mgr.y (mgr.34101) 572 : cluster 0 pgmap v1037: 713 pgs: 713 active+clean; 2.6 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 4.5 KiB/s wr, 53 op/s 2024-03-07T03:33:02.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:02 smithi148 bash[79819]: cluster 2024-03-07T03:33:01.427854+0000 mon.a (mon.0) 1443 : cluster 0 osdmap e847: 8 total, 8 up, 8 in 2024-03-07T03:33:02.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:02 smithi148 bash[79819]: cluster 2024-03-07T03:33:02.397761+0000 mon.a (mon.0) 1444 : cluster 0 osdmap e848: 8 total, 8 up, 8 in 2024-03-07T03:33:04.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:04 smithi148 bash[82222]: cluster 2024-03-07T03:33:02.852553+0000 mgr.y (mgr.34101) 573 : cluster 0 pgmap v1040: 713 pgs: 713 active+clean; 2.6 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 2 op/s 2024-03-07T03:33:04.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:04 smithi148 bash[82222]: cluster 2024-03-07T03:33:03.444132+0000 mon.a (mon.0) 1445 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2024-03-07T03:33:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:04 smithi148 bash[79819]: cluster 2024-03-07T03:33:02.852553+0000 mgr.y (mgr.34101) 573 : cluster 0 pgmap v1040: 713 pgs: 713 active+clean; 2.6 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 2 op/s 2024-03-07T03:33:04.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:04 smithi148 bash[79819]: cluster 2024-03-07T03:33:03.444132+0000 mon.a (mon.0) 1445 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2024-03-07T03:33:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:04 smithi195 bash[66542]: cluster 2024-03-07T03:33:02.852553+0000 mgr.y (mgr.34101) 573 : cluster 0 pgmap v1040: 713 pgs: 713 active+clean; 2.6 MiB data, 509 MiB used, 715 GiB / 715 GiB avail; 767 B/s rd, 2 op/s 2024-03-07T03:33:04.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:04 smithi195 bash[66542]: cluster 2024-03-07T03:33:03.444132+0000 mon.a (mon.0) 1445 : cluster 0 osdmap e849: 8 total, 8 up, 8 in 2024-03-07T03:33:05.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:05 smithi148 bash[79819]: cluster 2024-03-07T03:33:04.459721+0000 mon.a (mon.0) 1446 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2024-03-07T03:33:05.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:05 smithi148 bash[82222]: cluster 2024-03-07T03:33:04.459721+0000 mon.a (mon.0) 1446 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2024-03-07T03:33:05.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:05 smithi195 bash[66542]: cluster 2024-03-07T03:33:04.459721+0000 mon.a (mon.0) 1446 : cluster 0 osdmap e850: 8 total, 8 up, 8 in 2024-03-07T03:33:06.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:06 smithi148 bash[79819]: cluster 2024-03-07T03:33:04.853721+0000 mgr.y (mgr.34101) 574 : cluster 0 pgmap v1043: 649 pgs: 649 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 1 op/s 2024-03-07T03:33:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:06 smithi148 bash[79819]: cluster 2024-03-07T03:33:05.470770+0000 mon.a (mon.0) 1447 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2024-03-07T03:33:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:06 smithi148 bash[82222]: cluster 2024-03-07T03:33:04.853721+0000 mgr.y (mgr.34101) 574 : cluster 0 pgmap v1043: 649 pgs: 649 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 1 op/s 2024-03-07T03:33:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:06 smithi148 bash[82222]: cluster 2024-03-07T03:33:05.470770+0000 mon.a (mon.0) 1447 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2024-03-07T03:33:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:06 smithi195 bash[66542]: cluster 2024-03-07T03:33:04.853721+0000 mgr.y (mgr.34101) 574 : cluster 0 pgmap v1043: 649 pgs: 649 active+clean; 2.6 MiB data, 514 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 1 op/s 2024-03-07T03:33:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:06 smithi195 bash[66542]: cluster 2024-03-07T03:33:05.470770+0000 mon.a (mon.0) 1447 : cluster 0 osdmap e851: 8 total, 8 up, 8 in 2024-03-07T03:33:07.479 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:33:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:33:07] "GET /metrics HTTP/1.1" 200 82390 "" "Prometheus/2.43.0" 2024-03-07T03:33:07.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:07 smithi148 bash[82222]: cluster 2024-03-07T03:33:06.473289+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2024-03-07T03:33:07.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:07 smithi148 bash[82222]: cluster 2024-03-07T03:33:07.403524+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2024-03-07T03:33:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:07 smithi148 bash[79819]: cluster 2024-03-07T03:33:06.473289+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2024-03-07T03:33:07.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:07 smithi148 bash[79819]: cluster 2024-03-07T03:33:07.403524+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2024-03-07T03:33:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:07 smithi195 bash[66542]: cluster 2024-03-07T03:33:06.473289+0000 mon.a (mon.0) 1448 : cluster 0 osdmap e852: 8 total, 8 up, 8 in 2024-03-07T03:33:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:07 smithi195 bash[66542]: cluster 2024-03-07T03:33:07.403524+0000 mon.a (mon.0) 1449 : cluster 0 osdmap e853: 8 total, 8 up, 8 in 2024-03-07T03:33:08.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:08 smithi148 bash[79819]: cluster 2024-03-07T03:33:06.854868+0000 mgr.y (mgr.34101) 575 : cluster 0 pgmap v1046: 585 pgs: 585 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:08.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:08 smithi148 bash[79819]: cluster 2024-03-07T03:33:08.406580+0000 mon.a (mon.0) 1450 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2024-03-07T03:33:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:08 smithi148 bash[82222]: cluster 2024-03-07T03:33:06.854868+0000 mgr.y (mgr.34101) 575 : cluster 0 pgmap v1046: 585 pgs: 585 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:08 smithi148 bash[82222]: cluster 2024-03-07T03:33:08.406580+0000 mon.a (mon.0) 1450 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2024-03-07T03:33:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:08 smithi195 bash[66542]: cluster 2024-03-07T03:33:06.854868+0000 mgr.y (mgr.34101) 575 : cluster 0 pgmap v1046: 585 pgs: 585 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:08 smithi195 bash[66542]: cluster 2024-03-07T03:33:08.406580+0000 mon.a (mon.0) 1450 : cluster 0 osdmap e854: 8 total, 8 up, 8 in 2024-03-07T03:33:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:10 smithi148 bash[79819]: cluster 2024-03-07T03:33:08.856836+0000 mgr.y (mgr.34101) 576 : cluster 0 pgmap v1049: 521 pgs: 521 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:10.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:10 smithi148 bash[79819]: cluster 2024-03-07T03:33:09.408702+0000 mon.a (mon.0) 1451 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2024-03-07T03:33:10.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:10 smithi148 bash[82222]: cluster 2024-03-07T03:33:08.856836+0000 mgr.y (mgr.34101) 576 : cluster 0 pgmap v1049: 521 pgs: 521 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:10.794 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:10 smithi148 bash[82222]: cluster 2024-03-07T03:33:09.408702+0000 mon.a (mon.0) 1451 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2024-03-07T03:33:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:10 smithi195 bash[66542]: cluster 2024-03-07T03:33:08.856836+0000 mgr.y (mgr.34101) 576 : cluster 0 pgmap v1049: 521 pgs: 521 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:10 smithi195 bash[66542]: cluster 2024-03-07T03:33:09.408702+0000 mon.a (mon.0) 1451 : cluster 0 osdmap e855: 8 total, 8 up, 8 in 2024-03-07T03:33:11.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:11 smithi148 bash[82222]: cluster 2024-03-07T03:33:10.409944+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2024-03-07T03:33:11.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:11 smithi148 bash[79819]: cluster 2024-03-07T03:33:10.409944+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2024-03-07T03:33:11.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:11 smithi195 bash[66542]: cluster 2024-03-07T03:33:10.409944+0000 mon.a (mon.0) 1452 : cluster 0 osdmap e856: 8 total, 8 up, 8 in 2024-03-07T03:33:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:33:12 smithi195 bash[59134]: ts=2024-03-07T03:33:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:12 smithi195 bash[66542]: cluster 2024-03-07T03:33:10.858337+0000 mgr.y (mgr.34101) 577 : cluster 0 pgmap v1052: 457 pgs: 457 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:12 smithi195 bash[66542]: cluster 2024-03-07T03:33:11.438853+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2024-03-07T03:33:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:12 smithi195 bash[66542]: audit 2024-03-07T03:33:12.328232+0000 mon.a (mon.0) 1454 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:12.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:12 smithi195 bash[66542]: cluster 2024-03-07T03:33:12.403207+0000 mon.a (mon.0) 1455 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2024-03-07T03:33:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[82222]: cluster 2024-03-07T03:33:10.858337+0000 mgr.y (mgr.34101) 577 : cluster 0 pgmap v1052: 457 pgs: 457 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[82222]: cluster 2024-03-07T03:33:11.438853+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2024-03-07T03:33:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[82222]: audit 2024-03-07T03:33:12.328232+0000 mon.a (mon.0) 1454 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:12.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[82222]: cluster 2024-03-07T03:33:12.403207+0000 mon.a (mon.0) 1455 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2024-03-07T03:33:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[79819]: cluster 2024-03-07T03:33:10.858337+0000 mgr.y (mgr.34101) 577 : cluster 0 pgmap v1052: 457 pgs: 457 active+clean; 2.6 MiB data, 518 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[79819]: cluster 2024-03-07T03:33:11.438853+0000 mon.a (mon.0) 1453 : cluster 0 osdmap e857: 8 total, 8 up, 8 in 2024-03-07T03:33:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[79819]: audit 2024-03-07T03:33:12.328232+0000 mon.a (mon.0) 1454 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:12 smithi148 bash[79819]: cluster 2024-03-07T03:33:12.403207+0000 mon.a (mon.0) 1455 : cluster 0 osdmap e858: 8 total, 8 up, 8 in 2024-03-07T03:33:14.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:14 smithi148 bash[79819]: cluster 2024-03-07T03:33:12.859517+0000 mgr.y (mgr.34101) 578 : cluster 0 pgmap v1055: 393 pgs: 393 active+clean; 2.6 MiB data, 519 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:14.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:14 smithi148 bash[79819]: cluster 2024-03-07T03:33:13.404974+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2024-03-07T03:33:14.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:14 smithi148 bash[82222]: cluster 2024-03-07T03:33:12.859517+0000 mgr.y (mgr.34101) 578 : cluster 0 pgmap v1055: 393 pgs: 393 active+clean; 2.6 MiB data, 519 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:14.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:14 smithi148 bash[82222]: cluster 2024-03-07T03:33:13.404974+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2024-03-07T03:33:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:14 smithi195 bash[66542]: cluster 2024-03-07T03:33:12.859517+0000 mgr.y (mgr.34101) 578 : cluster 0 pgmap v1055: 393 pgs: 393 active+clean; 2.6 MiB data, 519 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:14 smithi195 bash[66542]: cluster 2024-03-07T03:33:13.404974+0000 mon.a (mon.0) 1456 : cluster 0 osdmap e859: 8 total, 8 up, 8 in 2024-03-07T03:33:15.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:15 smithi148 bash[79819]: cluster 2024-03-07T03:33:14.416793+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2024-03-07T03:33:15.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:15 smithi148 bash[82222]: cluster 2024-03-07T03:33:14.416793+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2024-03-07T03:33:15.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:15 smithi195 bash[66542]: cluster 2024-03-07T03:33:14.416793+0000 mon.a (mon.0) 1457 : cluster 0 osdmap e860: 8 total, 8 up, 8 in 2024-03-07T03:33:16.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:16 smithi148 bash[79819]: cluster 2024-03-07T03:33:14.860498+0000 mgr.y (mgr.34101) 579 : cluster 0 pgmap v1058: 329 pgs: 329 active+clean; 587 KiB data, 523 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:16.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:16 smithi148 bash[79819]: cluster 2024-03-07T03:33:15.428033+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2024-03-07T03:33:16.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:16 smithi148 bash[82222]: cluster 2024-03-07T03:33:14.860498+0000 mgr.y (mgr.34101) 579 : cluster 0 pgmap v1058: 329 pgs: 329 active+clean; 587 KiB data, 523 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:16.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:16 smithi148 bash[82222]: cluster 2024-03-07T03:33:15.428033+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2024-03-07T03:33:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:16 smithi195 bash[66542]: cluster 2024-03-07T03:33:14.860498+0000 mgr.y (mgr.34101) 579 : cluster 0 pgmap v1058: 329 pgs: 329 active+clean; 587 KiB data, 523 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:16 smithi195 bash[66542]: cluster 2024-03-07T03:33:15.428033+0000 mon.a (mon.0) 1458 : cluster 0 osdmap e861: 8 total, 8 up, 8 in 2024-03-07T03:33:17.438 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:33:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:33:17] "GET /metrics HTTP/1.1" 200 60716 "" "Prometheus/2.43.0" 2024-03-07T03:33:17.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:17 smithi148 bash[79819]: cluster 2024-03-07T03:33:16.429165+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2024-03-07T03:33:17.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:17 smithi148 bash[79819]: audit 2024-03-07T03:33:16.656285+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.b", "id": [6, 3]}]: dispatch 2024-03-07T03:33:17.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:17 smithi148 bash[79819]: audit 2024-03-07T03:33:16.656730+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]: dispatch 2024-03-07T03:33:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:17 smithi148 bash[82222]: cluster 2024-03-07T03:33:16.429165+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2024-03-07T03:33:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:17 smithi148 bash[82222]: audit 2024-03-07T03:33:16.656285+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.b", "id": [6, 3]}]: dispatch 2024-03-07T03:33:17.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:17 smithi148 bash[82222]: audit 2024-03-07T03:33:16.656730+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]: dispatch 2024-03-07T03:33:17.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:17 smithi195 bash[66542]: cluster 2024-03-07T03:33:16.429165+0000 mon.a (mon.0) 1459 : cluster 0 osdmap e862: 8 total, 8 up, 8 in 2024-03-07T03:33:17.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:17 smithi195 bash[66542]: audit 2024-03-07T03:33:16.656285+0000 mon.a (mon.0) 1460 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.b", "id": [6, 3]}]: dispatch 2024-03-07T03:33:17.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:17 smithi195 bash[66542]: audit 2024-03-07T03:33:16.656730+0000 mon.a (mon.0) 1461 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]: dispatch 2024-03-07T03:33:18.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[79819]: cluster 2024-03-07T03:33:16.861366+0000 mgr.y (mgr.34101) 580 : cluster 0 pgmap v1061: 265 pgs: 265 active+clean; 587 KiB data, 541 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:18.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[79819]: audit 2024-03-07T03:33:17.448937+0000 mon.a (mon.0) 1462 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.b", "id": [6, 3]}]': finished 2024-03-07T03:33:18.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[79819]: audit 2024-03-07T03:33:17.449201+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]': finished 2024-03-07T03:33:18.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[79819]: cluster 2024-03-07T03:33:17.460072+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2024-03-07T03:33:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[82222]: cluster 2024-03-07T03:33:16.861366+0000 mgr.y (mgr.34101) 580 : cluster 0 pgmap v1061: 265 pgs: 265 active+clean; 587 KiB data, 541 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[82222]: audit 2024-03-07T03:33:17.448937+0000 mon.a (mon.0) 1462 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.b", "id": [6, 3]}]': finished 2024-03-07T03:33:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[82222]: audit 2024-03-07T03:33:17.449201+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]': finished 2024-03-07T03:33:18.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:18 smithi148 bash[82222]: cluster 2024-03-07T03:33:17.460072+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2024-03-07T03:33:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:18 smithi195 bash[66542]: cluster 2024-03-07T03:33:16.861366+0000 mgr.y (mgr.34101) 580 : cluster 0 pgmap v1061: 265 pgs: 265 active+clean; 587 KiB data, 541 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:18 smithi195 bash[66542]: audit 2024-03-07T03:33:17.448937+0000 mon.a (mon.0) 1462 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.b", "id": [6, 3]}]': finished 2024-03-07T03:33:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:18 smithi195 bash[66542]: audit 2024-03-07T03:33:17.449201+0000 mon.a (mon.0) 1463 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.11", "id": [7, 1]}]': finished 2024-03-07T03:33:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:18 smithi195 bash[66542]: cluster 2024-03-07T03:33:17.460072+0000 mon.a (mon.0) 1464 : cluster 0 osdmap e863: 8 total, 8 up, 8 in 2024-03-07T03:33:19.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:19 smithi148 bash[79819]: cluster 2024-03-07T03:33:18.472914+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2024-03-07T03:33:19.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:19 smithi148 bash[82222]: cluster 2024-03-07T03:33:18.472914+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2024-03-07T03:33:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:19 smithi195 bash[66542]: cluster 2024-03-07T03:33:18.472914+0000 mon.a (mon.0) 1465 : cluster 0 osdmap e864: 8 total, 8 up, 8 in 2024-03-07T03:33:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:20 smithi148 bash[79819]: cluster 2024-03-07T03:33:18.862571+0000 mgr.y (mgr.34101) 581 : cluster 0 pgmap v1064: 201 pgs: 1 activating, 200 active+clean; 587 KiB data, 549 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:20.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:20 smithi148 bash[79819]: cluster 2024-03-07T03:33:19.485825+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2024-03-07T03:33:20.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:20 smithi148 bash[82222]: cluster 2024-03-07T03:33:18.862571+0000 mgr.y (mgr.34101) 581 : cluster 0 pgmap v1064: 201 pgs: 1 activating, 200 active+clean; 587 KiB data, 549 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:20.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:20 smithi148 bash[82222]: cluster 2024-03-07T03:33:19.485825+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2024-03-07T03:33:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:20 smithi195 bash[66542]: cluster 2024-03-07T03:33:18.862571+0000 mgr.y (mgr.34101) 581 : cluster 0 pgmap v1064: 201 pgs: 1 activating, 200 active+clean; 587 KiB data, 549 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:20 smithi195 bash[66542]: cluster 2024-03-07T03:33:19.485825+0000 mon.a (mon.0) 1466 : cluster 0 osdmap e865: 8 total, 8 up, 8 in 2024-03-07T03:33:21.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:21 smithi148 bash[79819]: cluster 2024-03-07T03:33:20.480767+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2024-03-07T03:33:21.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:21 smithi148 bash[82222]: cluster 2024-03-07T03:33:20.480767+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2024-03-07T03:33:21.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:21 smithi195 bash[66542]: cluster 2024-03-07T03:33:20.480767+0000 mon.a (mon.0) 1467 : cluster 0 osdmap e866: 8 total, 8 up, 8 in 2024-03-07T03:33:22.515 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 168 tests from TestLibRBD (624352 ms total) 2024-03-07T03:33:22.516 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:33:22.516 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/0, where TypeParam = PlaintextUnderLUKS1 2024-03-07T03:33:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:33:22 smithi195 bash[59134]: ts=2024-03-07T03:33:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:22 smithi195 bash[66542]: cluster 2024-03-07T03:33:20.863263+0000 mgr.y (mgr.34101) 582 : cluster 0 pgmap v1067: 137 pgs: 1 activating, 136 active+clean; 586 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:22 smithi195 bash[66542]: cluster 2024-03-07T03:33:21.501006+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2024-03-07T03:33:22.566 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/0.Simple 2024-03-07T03:33:22.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:22 smithi148 bash[79819]: cluster 2024-03-07T03:33:20.863263+0000 mgr.y (mgr.34101) 582 : cluster 0 pgmap v1067: 137 pgs: 1 activating, 136 active+clean; 586 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:22.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:22 smithi148 bash[79819]: cluster 2024-03-07T03:33:21.501006+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2024-03-07T03:33:22.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:22 smithi148 bash[82222]: cluster 2024-03-07T03:33:20.863263+0000 mgr.y (mgr.34101) 582 : cluster 0 pgmap v1067: 137 pgs: 1 activating, 136 active+clean; 586 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:22.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:22 smithi148 bash[82222]: cluster 2024-03-07T03:33:21.501006+0000 mon.a (mon.0) 1468 : cluster 0 osdmap e867: 8 total, 8 up, 8 in 2024-03-07T03:33:23.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:23 smithi148 bash[79819]: cluster 2024-03-07T03:33:22.521894+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2024-03-07T03:33:23.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:23 smithi148 bash[82222]: cluster 2024-03-07T03:33:22.521894+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2024-03-07T03:33:23.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:23 smithi195 bash[66542]: cluster 2024-03-07T03:33:22.521894+0000 mon.a (mon.0) 1469 : cluster 0 osdmap e868: 8 total, 8 up, 8 in 2024-03-07T03:33:24.538 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:33:24.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[82222]: cluster 2024-03-07T03:33:22.863882+0000 mgr.y (mgr.34101) 583 : cluster 0 pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[82222]: cluster 2024-03-07T03:33:23.534447+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2024-03-07T03:33:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[82222]: audit 2024-03-07T03:33:23.536602+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.148:0/2366176825' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:33:24.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[82222]: audit 2024-03-07T03:33:23.540251+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:33:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[79819]: cluster 2024-03-07T03:33:22.863882+0000 mgr.y (mgr.34101) 583 : cluster 0 pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[79819]: cluster 2024-03-07T03:33:23.534447+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2024-03-07T03:33:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[79819]: audit 2024-03-07T03:33:23.536602+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.148:0/2366176825' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:33:24.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:24 smithi148 bash[79819]: audit 2024-03-07T03:33:23.540251+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:33:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:24 smithi195 bash[66542]: cluster 2024-03-07T03:33:22.863882+0000 mgr.y (mgr.34101) 583 : cluster 0 pgmap v1070: 73 pgs: 73 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:24 smithi195 bash[66542]: cluster 2024-03-07T03:33:23.534447+0000 mon.a (mon.0) 1470 : cluster 0 osdmap e869: 8 total, 8 up, 8 in 2024-03-07T03:33:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:24 smithi195 bash[66542]: audit 2024-03-07T03:33:23.536602+0000 mon.c (mon.2) 79 : audit 1 from='client.? 172.21.15.148:0/2366176825' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:33:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:24 smithi195 bash[66542]: audit 2024-03-07T03:33:23.540251+0000 mon.a (mon.0) 1471 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:33:25.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:25 smithi195 bash[66542]: audit 2024-03-07T03:33:24.529854+0000 mon.a (mon.0) 1472 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:33:25.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:25 smithi195 bash[66542]: cluster 2024-03-07T03:33:24.536814+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2024-03-07T03:33:26.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:25 smithi148 bash[79819]: audit 2024-03-07T03:33:24.529854+0000 mon.a (mon.0) 1472 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:33:26.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:25 smithi148 bash[79819]: cluster 2024-03-07T03:33:24.536814+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2024-03-07T03:33:26.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:25 smithi148 bash[82222]: audit 2024-03-07T03:33:24.529854+0000 mon.a (mon.0) 1472 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:33:26.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:25 smithi148 bash[82222]: cluster 2024-03-07T03:33:24.536814+0000 mon.a (mon.0) 1473 : cluster 0 osdmap e870: 8 total, 8 up, 8 in 2024-03-07T03:33:26.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:26 smithi195 bash[66542]: cluster 2024-03-07T03:33:24.864575+0000 mgr.y (mgr.34101) 584 : cluster 0 pgmap v1073: 105 pgs: 10 creating+peering, 22 unknown, 73 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:26.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:26 smithi195 bash[66542]: cluster 2024-03-07T03:33:25.561365+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2024-03-07T03:33:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:26 smithi148 bash[79819]: cluster 2024-03-07T03:33:24.864575+0000 mgr.y (mgr.34101) 584 : cluster 0 pgmap v1073: 105 pgs: 10 creating+peering, 22 unknown, 73 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:27.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:26 smithi148 bash[79819]: cluster 2024-03-07T03:33:25.561365+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2024-03-07T03:33:27.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:26 smithi148 bash[82222]: cluster 2024-03-07T03:33:24.864575+0000 mgr.y (mgr.34101) 584 : cluster 0 pgmap v1073: 105 pgs: 10 creating+peering, 22 unknown, 73 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:27.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:26 smithi148 bash[82222]: cluster 2024-03-07T03:33:25.561365+0000 mon.a (mon.0) 1474 : cluster 0 osdmap e871: 8 total, 8 up, 8 in 2024-03-07T03:33:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:33:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:33:27] "GET /metrics HTTP/1.1" 200 60716 "" "Prometheus/2.43.0" 2024-03-07T03:33:27.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:27 smithi195 bash[66542]: cluster 2024-03-07T03:33:26.562677+0000 mon.a (mon.0) 1475 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2024-03-07T03:33:27.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:27 smithi195 bash[66542]: audit 2024-03-07T03:33:27.327987+0000 mon.a (mon.0) 1476 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:28.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:27 smithi148 bash[82222]: cluster 2024-03-07T03:33:26.562677+0000 mon.a (mon.0) 1475 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2024-03-07T03:33:28.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:27 smithi148 bash[82222]: audit 2024-03-07T03:33:27.327987+0000 mon.a (mon.0) 1476 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:27 smithi148 bash[79819]: cluster 2024-03-07T03:33:26.562677+0000 mon.a (mon.0) 1475 : cluster 0 osdmap e872: 8 total, 8 up, 8 in 2024-03-07T03:33:28.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:27 smithi148 bash[79819]: audit 2024-03-07T03:33:27.327987+0000 mon.a (mon.0) 1476 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:29.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:28 smithi148 bash[82222]: cluster 2024-03-07T03:33:26.865214+0000 mgr.y (mgr.34101) 585 : cluster 0 pgmap v1076: 105 pgs: 10 creating+peering, 9 unknown, 86 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:29.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:28 smithi148 bash[82222]: cluster 2024-03-07T03:33:27.589361+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2024-03-07T03:33:29.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:28 smithi148 bash[79819]: cluster 2024-03-07T03:33:26.865214+0000 mgr.y (mgr.34101) 585 : cluster 0 pgmap v1076: 105 pgs: 10 creating+peering, 9 unknown, 86 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:29.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:28 smithi148 bash[79819]: cluster 2024-03-07T03:33:27.589361+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2024-03-07T03:33:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:28 smithi195 bash[66542]: cluster 2024-03-07T03:33:26.865214+0000 mgr.y (mgr.34101) 585 : cluster 0 pgmap v1076: 105 pgs: 10 creating+peering, 9 unknown, 86 active+clean; 582 KiB data, 550 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:33:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:28 smithi195 bash[66542]: cluster 2024-03-07T03:33:27.589361+0000 mon.a (mon.0) 1477 : cluster 0 osdmap e873: 8 total, 8 up, 8 in 2024-03-07T03:33:31.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:30 smithi148 bash[82222]: cluster 2024-03-07T03:33:28.866345+0000 mgr.y (mgr.34101) 586 : cluster 0 pgmap v1078: 105 pgs: 4 creating+peering, 101 active+clean; 9.6 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-03-07T03:33:31.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:30 smithi148 bash[82222]: audit 2024-03-07T03:33:29.751698+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:33:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:30 smithi148 bash[79819]: cluster 2024-03-07T03:33:28.866345+0000 mgr.y (mgr.34101) 586 : cluster 0 pgmap v1078: 105 pgs: 4 creating+peering, 101 active+clean; 9.6 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-03-07T03:33:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:30 smithi148 bash[79819]: audit 2024-03-07T03:33:29.751698+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:33:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:30 smithi195 bash[66542]: cluster 2024-03-07T03:33:28.866345+0000 mgr.y (mgr.34101) 586 : cluster 0 pgmap v1078: 105 pgs: 4 creating+peering, 101 active+clean; 9.6 MiB data, 593 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 2.1 MiB/s wr, 20 op/s 2024-03-07T03:33:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:30 smithi195 bash[66542]: audit 2024-03-07T03:33:29.751698+0000 mon.a (mon.0) 1478 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:33:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:33:32 smithi195 bash[59134]: ts=2024-03-07T03:33:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:33.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:32 smithi148 bash[79819]: cluster 2024-03-07T03:33:30.867291+0000 mgr.y (mgr.34101) 587 : cluster 0 pgmap v1079: 105 pgs: 105 active+clean; 22 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 3.5 MiB/s wr, 29 op/s 2024-03-07T03:33:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:32 smithi148 bash[82222]: cluster 2024-03-07T03:33:30.867291+0000 mgr.y (mgr.34101) 587 : cluster 0 pgmap v1079: 105 pgs: 105 active+clean; 22 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 3.5 MiB/s wr, 29 op/s 2024-03-07T03:33:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:32 smithi195 bash[66542]: cluster 2024-03-07T03:33:30.867291+0000 mgr.y (mgr.34101) 587 : cluster 0 pgmap v1079: 105 pgs: 105 active+clean; 22 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 3.5 MiB/s wr, 29 op/s 2024-03-07T03:33:35.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:34 smithi148 bash[79819]: cluster 2024-03-07T03:33:32.868192+0000 mgr.y (mgr.34101) 588 : cluster 0 pgmap v1080: 105 pgs: 105 active+clean; 26 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 3.4 MiB/s wr, 50 op/s 2024-03-07T03:33:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:34 smithi148 bash[82222]: cluster 2024-03-07T03:33:32.868192+0000 mgr.y (mgr.34101) 588 : cluster 0 pgmap v1080: 105 pgs: 105 active+clean; 26 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 3.4 MiB/s wr, 50 op/s 2024-03-07T03:33:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:34 smithi195 bash[66542]: cluster 2024-03-07T03:33:32.868192+0000 mgr.y (mgr.34101) 588 : cluster 0 pgmap v1080: 105 pgs: 105 active+clean; 26 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 3.4 MiB/s wr, 50 op/s 2024-03-07T03:33:36.234 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/0.Simple (13667 ms) 2024-03-07T03:33:36.234 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/0.Grow 2024-03-07T03:33:36.234 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: cluster 2024-03-07T03:33:34.868995+0000 mgr.y (mgr.34101) 589 : cluster 0 pgmap v1081: 105 pgs: 105 active+clean; 34 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 4.0 MiB/s wr, 52 op/s 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: audit 2024-03-07T03:33:35.538731+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: audit 2024-03-07T03:33:35.544383+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: audit 2024-03-07T03:33:35.793695+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: audit 2024-03-07T03:33:35.804778+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: audit 2024-03-07T03:33:36.440887+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: audit 2024-03-07T03:33:36.441954+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:33:36.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[79819]: audit 2024-03-07T03:33:36.449854+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: cluster 2024-03-07T03:33:34.868995+0000 mgr.y (mgr.34101) 589 : cluster 0 pgmap v1081: 105 pgs: 105 active+clean; 34 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 4.0 MiB/s wr, 52 op/s 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: audit 2024-03-07T03:33:35.538731+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: audit 2024-03-07T03:33:35.544383+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: audit 2024-03-07T03:33:35.793695+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: audit 2024-03-07T03:33:35.804778+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: audit 2024-03-07T03:33:36.440887+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: audit 2024-03-07T03:33:36.441954+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:33:36.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:36 smithi148 bash[82222]: audit 2024-03-07T03:33:36.449854+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: cluster 2024-03-07T03:33:34.868995+0000 mgr.y (mgr.34101) 589 : cluster 0 pgmap v1081: 105 pgs: 105 active+clean; 34 MiB data, 630 MiB used, 715 GiB / 715 GiB avail; 3.0 MiB/s rd, 4.0 MiB/s wr, 52 op/s 2024-03-07T03:33:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: audit 2024-03-07T03:33:35.538731+0000 mon.a (mon.0) 1479 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: audit 2024-03-07T03:33:35.544383+0000 mon.a (mon.0) 1480 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: audit 2024-03-07T03:33:35.793695+0000 mon.a (mon.0) 1481 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: audit 2024-03-07T03:33:35.804778+0000 mon.a (mon.0) 1482 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: audit 2024-03-07T03:33:36.440887+0000 mon.a (mon.0) 1483 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:33:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: audit 2024-03-07T03:33:36.441954+0000 mon.a (mon.0) 1484 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:33:36.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:36 smithi195 bash[66542]: audit 2024-03-07T03:33:36.449854+0000 mon.a (mon.0) 1485 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:33:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:33:37] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.43.0" 2024-03-07T03:33:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:37 smithi195 bash[66542]: cluster 2024-03-07T03:33:36.811982+0000 mon.a (mon.0) 1486 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2024-03-07T03:33:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:37 smithi195 bash[66542]: cluster 2024-03-07T03:33:36.869618+0000 mgr.y (mgr.34101) 590 : cluster 0 pgmap v1083: 105 pgs: 105 active+clean; 47 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 5.4 MiB/s wr, 53 op/s 2024-03-07T03:33:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:37 smithi195 bash[66542]: cluster 2024-03-07T03:33:37.418582+0000 mon.a (mon.0) 1487 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2024-03-07T03:33:38.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:37 smithi148 bash[79819]: cluster 2024-03-07T03:33:36.811982+0000 mon.a (mon.0) 1486 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2024-03-07T03:33:38.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:37 smithi148 bash[79819]: cluster 2024-03-07T03:33:36.869618+0000 mgr.y (mgr.34101) 590 : cluster 0 pgmap v1083: 105 pgs: 105 active+clean; 47 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 5.4 MiB/s wr, 53 op/s 2024-03-07T03:33:38.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:37 smithi148 bash[79819]: cluster 2024-03-07T03:33:37.418582+0000 mon.a (mon.0) 1487 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2024-03-07T03:33:38.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:37 smithi148 bash[82222]: cluster 2024-03-07T03:33:36.811982+0000 mon.a (mon.0) 1486 : cluster 0 osdmap e874: 8 total, 8 up, 8 in 2024-03-07T03:33:38.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:37 smithi148 bash[82222]: cluster 2024-03-07T03:33:36.869618+0000 mgr.y (mgr.34101) 590 : cluster 0 pgmap v1083: 105 pgs: 105 active+clean; 47 MiB data, 672 MiB used, 715 GiB / 715 GiB avail; 6.3 MiB/s rd, 5.4 MiB/s wr, 53 op/s 2024-03-07T03:33:38.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:37 smithi148 bash[82222]: cluster 2024-03-07T03:33:37.418582+0000 mon.a (mon.0) 1487 : cluster 0 osdmap e875: 8 total, 8 up, 8 in 2024-03-07T03:33:39.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:39 smithi148 bash[79819]: audit 2024-03-07T03:33:38.523759+0000 mon.a (mon.0) 1488 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:39.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:39 smithi148 bash[82222]: audit 2024-03-07T03:33:38.523759+0000 mon.a (mon.0) 1488 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:39.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:39 smithi195 bash[66542]: audit 2024-03-07T03:33:38.523759+0000 mon.a (mon.0) 1488 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:33:40.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:40 smithi148 bash[79819]: cluster 2024-03-07T03:33:38.870682+0000 mgr.y (mgr.34101) 591 : cluster 0 pgmap v1085: 105 pgs: 105 active+clean; 68 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 6.3 MiB/s wr, 74 op/s 2024-03-07T03:33:40.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:40 smithi148 bash[82222]: cluster 2024-03-07T03:33:38.870682+0000 mgr.y (mgr.34101) 591 : cluster 0 pgmap v1085: 105 pgs: 105 active+clean; 68 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 6.3 MiB/s wr, 74 op/s 2024-03-07T03:33:40.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:40 smithi195 bash[66542]: cluster 2024-03-07T03:33:38.870682+0000 mgr.y (mgr.34101) 591 : cluster 0 pgmap v1085: 105 pgs: 105 active+clean; 68 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 6.3 MiB/s wr, 74 op/s 2024-03-07T03:33:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:33:42 smithi195 bash[59134]: ts=2024-03-07T03:33:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:42.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:42 smithi195 bash[66542]: cluster 2024-03-07T03:33:40.871716+0000 mgr.y (mgr.34101) 592 : cluster 0 pgmap v1086: 105 pgs: 105 active+clean; 76 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 77 op/s 2024-03-07T03:33:42.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:42 smithi195 bash[66542]: audit 2024-03-07T03:33:42.328571+0000 mon.a (mon.0) 1489 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:42 smithi148 bash[79819]: cluster 2024-03-07T03:33:40.871716+0000 mgr.y (mgr.34101) 592 : cluster 0 pgmap v1086: 105 pgs: 105 active+clean; 76 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 77 op/s 2024-03-07T03:33:42.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:42 smithi148 bash[79819]: audit 2024-03-07T03:33:42.328571+0000 mon.a (mon.0) 1489 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:42 smithi148 bash[82222]: cluster 2024-03-07T03:33:40.871716+0000 mgr.y (mgr.34101) 592 : cluster 0 pgmap v1086: 105 pgs: 105 active+clean; 76 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 77 op/s 2024-03-07T03:33:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:42 smithi148 bash[82222]: audit 2024-03-07T03:33:42.328571+0000 mon.a (mon.0) 1489 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:44.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:44 smithi148 bash[82222]: cluster 2024-03-07T03:33:42.872516+0000 mgr.y (mgr.34101) 593 : cluster 0 pgmap v1087: 105 pgs: 105 active+clean; 80 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.3 MiB/s wr, 73 op/s 2024-03-07T03:33:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:44 smithi148 bash[79819]: cluster 2024-03-07T03:33:42.872516+0000 mgr.y (mgr.34101) 593 : cluster 0 pgmap v1087: 105 pgs: 105 active+clean; 80 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.3 MiB/s wr, 73 op/s 2024-03-07T03:33:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:44 smithi195 bash[66542]: cluster 2024-03-07T03:33:42.872516+0000 mgr.y (mgr.34101) 593 : cluster 0 pgmap v1087: 105 pgs: 105 active+clean; 80 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.3 MiB/s wr, 73 op/s 2024-03-07T03:33:45.313 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/0.Grow (9079 ms) 2024-03-07T03:33:45.313 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/0.Shrink 2024-03-07T03:33:45.313 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:33:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:46 smithi148 bash[82222]: cluster 2024-03-07T03:33:44.873619+0000 mgr.y (mgr.34101) 594 : cluster 0 pgmap v1088: 105 pgs: 105 active+clean; 96 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.6 MiB/s wr, 80 op/s 2024-03-07T03:33:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:46 smithi148 bash[82222]: cluster 2024-03-07T03:33:45.549300+0000 mon.a (mon.0) 1490 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2024-03-07T03:33:46.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:46 smithi148 bash[79819]: cluster 2024-03-07T03:33:44.873619+0000 mgr.y (mgr.34101) 594 : cluster 0 pgmap v1088: 105 pgs: 105 active+clean; 96 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.6 MiB/s wr, 80 op/s 2024-03-07T03:33:46.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:46 smithi148 bash[79819]: cluster 2024-03-07T03:33:45.549300+0000 mon.a (mon.0) 1490 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2024-03-07T03:33:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:46 smithi195 bash[66542]: cluster 2024-03-07T03:33:44.873619+0000 mgr.y (mgr.34101) 594 : cluster 0 pgmap v1088: 105 pgs: 105 active+clean; 96 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.6 MiB/s wr, 80 op/s 2024-03-07T03:33:46.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:46 smithi195 bash[66542]: cluster 2024-03-07T03:33:45.549300+0000 mon.a (mon.0) 1490 : cluster 0 osdmap e876: 8 total, 8 up, 8 in 2024-03-07T03:33:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:33:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:33:47] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.43.0" 2024-03-07T03:33:48.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:48 smithi148 bash[79819]: cluster 2024-03-07T03:33:46.874440+0000 mgr.y (mgr.34101) 595 : cluster 0 pgmap v1090: 105 pgs: 105 active+clean; 109 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 7.0 MiB/s wr, 84 op/s 2024-03-07T03:33:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:48 smithi148 bash[82222]: cluster 2024-03-07T03:33:46.874440+0000 mgr.y (mgr.34101) 595 : cluster 0 pgmap v1090: 105 pgs: 105 active+clean; 109 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 7.0 MiB/s wr, 84 op/s 2024-03-07T03:33:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:48 smithi195 bash[66542]: cluster 2024-03-07T03:33:46.874440+0000 mgr.y (mgr.34101) 595 : cluster 0 pgmap v1090: 105 pgs: 105 active+clean; 109 MiB data, 866 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 7.0 MiB/s wr, 84 op/s 2024-03-07T03:33:50.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:50 smithi148 bash[79819]: cluster 2024-03-07T03:33:48.875605+0000 mgr.y (mgr.34101) 596 : cluster 0 pgmap v1091: 105 pgs: 105 active+clean; 122 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 85 op/s 2024-03-07T03:33:50.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:50 smithi148 bash[82222]: cluster 2024-03-07T03:33:48.875605+0000 mgr.y (mgr.34101) 596 : cluster 0 pgmap v1091: 105 pgs: 105 active+clean; 122 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 85 op/s 2024-03-07T03:33:50.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:50 smithi195 bash[66542]: cluster 2024-03-07T03:33:48.875605+0000 mgr.y (mgr.34101) 596 : cluster 0 pgmap v1091: 105 pgs: 105 active+clean; 122 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 85 op/s 2024-03-07T03:33:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:52 smithi195 bash[66542]: cluster 2024-03-07T03:33:50.876622+0000 mgr.y (mgr.34101) 597 : cluster 0 pgmap v1092: 105 pgs: 105 active+clean; 138 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.6 MiB/s wr, 67 op/s 2024-03-07T03:33:52.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:33:52 smithi195 bash[59134]: ts=2024-03-07T03:33:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:33:53.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:52 smithi148 bash[79819]: cluster 2024-03-07T03:33:50.876622+0000 mgr.y (mgr.34101) 597 : cluster 0 pgmap v1092: 105 pgs: 105 active+clean; 138 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.6 MiB/s wr, 67 op/s 2024-03-07T03:33:53.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:52 smithi148 bash[82222]: cluster 2024-03-07T03:33:50.876622+0000 mgr.y (mgr.34101) 597 : cluster 0 pgmap v1092: 105 pgs: 105 active+clean; 138 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.6 MiB/s wr, 67 op/s 2024-03-07T03:33:54.108 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/0.Shrink (8795 ms) 2024-03-07T03:33:54.108 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/0.ShrinkToOne 2024-03-07T03:33:54.108 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:33:54.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:54 smithi195 bash[66542]: cluster 2024-03-07T03:33:52.877079+0000 mgr.y (mgr.34101) 598 : cluster 0 pgmap v1093: 105 pgs: 105 active+clean; 138 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 66 op/s 2024-03-07T03:33:55.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:54 smithi148 bash[79819]: cluster 2024-03-07T03:33:52.877079+0000 mgr.y (mgr.34101) 598 : cluster 0 pgmap v1093: 105 pgs: 105 active+clean; 138 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 66 op/s 2024-03-07T03:33:55.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:54 smithi148 bash[82222]: cluster 2024-03-07T03:33:52.877079+0000 mgr.y (mgr.34101) 598 : cluster 0 pgmap v1093: 105 pgs: 105 active+clean; 138 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 66 op/s 2024-03-07T03:33:56.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:55 smithi148 bash[79819]: cluster 2024-03-07T03:33:54.575490+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2024-03-07T03:33:56.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:55 smithi148 bash[82222]: cluster 2024-03-07T03:33:54.575490+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2024-03-07T03:33:56.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:55 smithi195 bash[66542]: cluster 2024-03-07T03:33:54.575490+0000 mon.a (mon.0) 1491 : cluster 0 osdmap e877: 8 total, 8 up, 8 in 2024-03-07T03:33:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:56 smithi148 bash[79819]: cluster 2024-03-07T03:33:54.877501+0000 mgr.y (mgr.34101) 599 : cluster 0 pgmap v1095: 105 pgs: 105 active+clean; 154 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-03-07T03:33:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:56 smithi148 bash[82222]: cluster 2024-03-07T03:33:54.877501+0000 mgr.y (mgr.34101) 599 : cluster 0 pgmap v1095: 105 pgs: 105 active+clean; 154 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-03-07T03:33:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:56 smithi195 bash[66542]: cluster 2024-03-07T03:33:54.877501+0000 mgr.y (mgr.34101) 599 : cluster 0 pgmap v1095: 105 pgs: 105 active+clean; 154 MiB data, 979 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-03-07T03:33:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:33:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:33:57] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.43.0" 2024-03-07T03:33:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:57 smithi148 bash[79819]: audit 2024-03-07T03:33:57.328817+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:57 smithi148 bash[82222]: audit 2024-03-07T03:33:57.328817+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:57 smithi195 bash[66542]: audit 2024-03-07T03:33:57.328817+0000 mon.a (mon.0) 1492 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:33:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:33:58 smithi148 bash[79819]: cluster 2024-03-07T03:33:56.878006+0000 mgr.y (mgr.34101) 600 : cluster 0 pgmap v1096: 105 pgs: 105 active+clean; 180 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.5 MiB/s wr, 67 op/s 2024-03-07T03:33:59.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:33:58 smithi148 bash[82222]: cluster 2024-03-07T03:33:56.878006+0000 mgr.y (mgr.34101) 600 : cluster 0 pgmap v1096: 105 pgs: 105 active+clean; 180 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.5 MiB/s wr, 67 op/s 2024-03-07T03:33:59.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:33:58 smithi195 bash[66542]: cluster 2024-03-07T03:33:56.878006+0000 mgr.y (mgr.34101) 600 : cluster 0 pgmap v1096: 105 pgs: 105 active+clean; 180 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.5 MiB/s wr, 67 op/s 2024-03-07T03:34:01.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:00 smithi148 bash[79819]: cluster 2024-03-07T03:33:58.879010+0000 mgr.y (mgr.34101) 601 : cluster 0 pgmap v1097: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:34:01.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:00 smithi148 bash[82222]: cluster 2024-03-07T03:33:58.879010+0000 mgr.y (mgr.34101) 601 : cluster 0 pgmap v1097: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:34:01.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:00 smithi195 bash[66542]: cluster 2024-03-07T03:33:58.879010+0000 mgr.y (mgr.34101) 601 : cluster 0 pgmap v1097: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:34:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:34:02 smithi195 bash[59134]: ts=2024-03-07T03:34:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:03.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:02 smithi148 bash[79819]: cluster 2024-03-07T03:34:00.879970+0000 mgr.y (mgr.34101) 602 : cluster 0 pgmap v1098: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 69 op/s 2024-03-07T03:34:03.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:02 smithi148 bash[82222]: cluster 2024-03-07T03:34:00.879970+0000 mgr.y (mgr.34101) 602 : cluster 0 pgmap v1098: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 69 op/s 2024-03-07T03:34:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:02 smithi195 bash[66542]: cluster 2024-03-07T03:34:00.879970+0000 mgr.y (mgr.34101) 602 : cluster 0 pgmap v1098: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 69 op/s 2024-03-07T03:34:03.073 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/0.ShrinkToOne (8965 ms) 2024-03-07T03:34:03.073 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/0.ShrinkToOneAfterSnapshot 2024-03-07T03:34:03.073 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:34:05.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:04 smithi148 bash[79819]: cluster 2024-03-07T03:34:02.880730+0000 mgr.y (mgr.34101) 603 : cluster 0 pgmap v1099: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 68 op/s 2024-03-07T03:34:05.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:04 smithi148 bash[79819]: cluster 2024-03-07T03:34:03.611375+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2024-03-07T03:34:05.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:04 smithi148 bash[82222]: cluster 2024-03-07T03:34:02.880730+0000 mgr.y (mgr.34101) 603 : cluster 0 pgmap v1099: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 68 op/s 2024-03-07T03:34:05.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:04 smithi148 bash[82222]: cluster 2024-03-07T03:34:03.611375+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2024-03-07T03:34:05.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:04 smithi195 bash[66542]: cluster 2024-03-07T03:34:02.880730+0000 mgr.y (mgr.34101) 603 : cluster 0 pgmap v1099: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 68 op/s 2024-03-07T03:34:05.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:04 smithi195 bash[66542]: cluster 2024-03-07T03:34:03.611375+0000 mon.a (mon.0) 1493 : cluster 0 osdmap e878: 8 total, 8 up, 8 in 2024-03-07T03:34:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:06 smithi148 bash[79819]: cluster 2024-03-07T03:34:04.881280+0000 mgr.y (mgr.34101) 604 : cluster 0 pgmap v1101: 105 pgs: 105 active+clean; 216 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.4 MiB/s wr, 74 op/s 2024-03-07T03:34:07.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:06 smithi148 bash[82222]: cluster 2024-03-07T03:34:04.881280+0000 mgr.y (mgr.34101) 604 : cluster 0 pgmap v1101: 105 pgs: 105 active+clean; 216 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.4 MiB/s wr, 74 op/s 2024-03-07T03:34:07.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:06 smithi195 bash[66542]: cluster 2024-03-07T03:34:04.881280+0000 mgr.y (mgr.34101) 604 : cluster 0 pgmap v1101: 105 pgs: 105 active+clean; 216 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 7.4 MiB/s wr, 74 op/s 2024-03-07T03:34:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:34:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:34:07] "GET /metrics HTTP/1.1" 200 36726 "" "Prometheus/2.43.0" 2024-03-07T03:34:09.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:08 smithi148 bash[79819]: cluster 2024-03-07T03:34:06.881802+0000 mgr.y (mgr.34101) 605 : cluster 0 pgmap v1102: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.9 MiB/s wr, 73 op/s 2024-03-07T03:34:09.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:08 smithi148 bash[82222]: cluster 2024-03-07T03:34:06.881802+0000 mgr.y (mgr.34101) 605 : cluster 0 pgmap v1102: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.9 MiB/s wr, 73 op/s 2024-03-07T03:34:09.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:08 smithi195 bash[66542]: cluster 2024-03-07T03:34:06.881802+0000 mgr.y (mgr.34101) 605 : cluster 0 pgmap v1102: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.9 MiB/s wr, 73 op/s 2024-03-07T03:34:11.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:10 smithi148 bash[79819]: cluster 2024-03-07T03:34:08.882827+0000 mgr.y (mgr.34101) 606 : cluster 0 pgmap v1103: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 954 KiB/s rd, 4.1 MiB/s wr, 65 op/s 2024-03-07T03:34:11.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:10 smithi148 bash[82222]: cluster 2024-03-07T03:34:08.882827+0000 mgr.y (mgr.34101) 606 : cluster 0 pgmap v1103: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 954 KiB/s rd, 4.1 MiB/s wr, 65 op/s 2024-03-07T03:34:11.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:10 smithi195 bash[66542]: cluster 2024-03-07T03:34:08.882827+0000 mgr.y (mgr.34101) 606 : cluster 0 pgmap v1103: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 954 KiB/s rd, 4.1 MiB/s wr, 65 op/s 2024-03-07T03:34:12.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:11 smithi148 bash[79819]: cluster 2024-03-07T03:34:10.634357+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2024-03-07T03:34:12.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:11 smithi148 bash[82222]: cluster 2024-03-07T03:34:10.634357+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2024-03-07T03:34:12.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:11 smithi195 bash[66542]: cluster 2024-03-07T03:34:10.634357+0000 mon.a (mon.0) 1494 : cluster 0 osdmap e879: 8 total, 8 up, 8 in 2024-03-07T03:34:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:34:12 smithi195 bash[59134]: ts=2024-03-07T03:34:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:12 smithi148 bash[79819]: cluster 2024-03-07T03:34:10.883531+0000 mgr.y (mgr.34101) 607 : cluster 0 pgmap v1105: 105 pgs: 105 active+clean; 225 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.6 MiB/s wr, 75 op/s 2024-03-07T03:34:13.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:12 smithi148 bash[79819]: audit 2024-03-07T03:34:12.329391+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:13.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:12 smithi148 bash[82222]: cluster 2024-03-07T03:34:10.883531+0000 mgr.y (mgr.34101) 607 : cluster 0 pgmap v1105: 105 pgs: 105 active+clean; 225 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.6 MiB/s wr, 75 op/s 2024-03-07T03:34:13.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:12 smithi148 bash[82222]: audit 2024-03-07T03:34:12.329391+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:13.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:12 smithi195 bash[66542]: cluster 2024-03-07T03:34:10.883531+0000 mgr.y (mgr.34101) 607 : cluster 0 pgmap v1105: 105 pgs: 105 active+clean; 225 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 5.6 MiB/s wr, 75 op/s 2024-03-07T03:34:13.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:12 smithi195 bash[66542]: audit 2024-03-07T03:34:12.329391+0000 mon.a (mon.0) 1495 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:13.102 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/0.ShrinkToOneAfterSnapshot (10029 ms) 2024-03-07T03:34:13.102 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/0.MinOverlap 2024-03-07T03:34:13.102 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:34:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:14 smithi148 bash[79819]: cluster 2024-03-07T03:34:12.884274+0000 mgr.y (mgr.34101) 608 : cluster 0 pgmap v1106: 105 pgs: 105 active+clean; 230 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 545 KiB/s rd, 2.8 MiB/s wr, 55 op/s 2024-03-07T03:34:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:14 smithi148 bash[79819]: cluster 2024-03-07T03:34:13.651282+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2024-03-07T03:34:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:14 smithi148 bash[82222]: cluster 2024-03-07T03:34:12.884274+0000 mgr.y (mgr.34101) 608 : cluster 0 pgmap v1106: 105 pgs: 105 active+clean; 230 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 545 KiB/s rd, 2.8 MiB/s wr, 55 op/s 2024-03-07T03:34:15.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:14 smithi148 bash[82222]: cluster 2024-03-07T03:34:13.651282+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2024-03-07T03:34:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:14 smithi195 bash[66542]: cluster 2024-03-07T03:34:12.884274+0000 mgr.y (mgr.34101) 608 : cluster 0 pgmap v1106: 105 pgs: 105 active+clean; 230 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 545 KiB/s rd, 2.8 MiB/s wr, 55 op/s 2024-03-07T03:34:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:14 smithi195 bash[66542]: cluster 2024-03-07T03:34:13.651282+0000 mon.a (mon.0) 1496 : cluster 0 osdmap e880: 8 total, 8 up, 8 in 2024-03-07T03:34:17.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:16 smithi148 bash[79819]: cluster 2024-03-07T03:34:14.885124+0000 mgr.y (mgr.34101) 609 : cluster 0 pgmap v1108: 105 pgs: 105 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 7.6 MiB/s wr, 95 op/s 2024-03-07T03:34:17.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:16 smithi148 bash[82222]: cluster 2024-03-07T03:34:14.885124+0000 mgr.y (mgr.34101) 609 : cluster 0 pgmap v1108: 105 pgs: 105 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 7.6 MiB/s wr, 95 op/s 2024-03-07T03:34:17.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:16 smithi195 bash[66542]: cluster 2024-03-07T03:34:14.885124+0000 mgr.y (mgr.34101) 609 : cluster 0 pgmap v1108: 105 pgs: 105 active+clean; 282 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 7.6 MiB/s wr, 95 op/s 2024-03-07T03:34:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:34:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:34:17] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:34:19.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:18 smithi148 bash[79819]: cluster 2024-03-07T03:34:16.885974+0000 mgr.y (mgr.34101) 610 : cluster 0 pgmap v1109: 105 pgs: 105 active+clean; 307 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 11 MiB/s wr, 81 op/s 2024-03-07T03:34:19.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:18 smithi148 bash[82222]: cluster 2024-03-07T03:34:16.885974+0000 mgr.y (mgr.34101) 610 : cluster 0 pgmap v1109: 105 pgs: 105 active+clean; 307 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 11 MiB/s wr, 81 op/s 2024-03-07T03:34:19.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:18 smithi195 bash[66542]: cluster 2024-03-07T03:34:16.885974+0000 mgr.y (mgr.34101) 610 : cluster 0 pgmap v1109: 105 pgs: 105 active+clean; 307 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 11 MiB/s wr, 81 op/s 2024-03-07T03:34:21.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:20 smithi148 bash[79819]: cluster 2024-03-07T03:34:18.886970+0000 mgr.y (mgr.34101) 611 : cluster 0 pgmap v1110: 105 pgs: 105 active+clean; 311 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 10 MiB/s wr, 85 op/s 2024-03-07T03:34:21.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:20 smithi148 bash[82222]: cluster 2024-03-07T03:34:18.886970+0000 mgr.y (mgr.34101) 611 : cluster 0 pgmap v1110: 105 pgs: 105 active+clean; 311 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 10 MiB/s wr, 85 op/s 2024-03-07T03:34:21.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:20 smithi195 bash[66542]: cluster 2024-03-07T03:34:18.886970+0000 mgr.y (mgr.34101) 611 : cluster 0 pgmap v1110: 105 pgs: 105 active+clean; 311 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 10 MiB/s wr, 85 op/s 2024-03-07T03:34:22.187 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/0.MinOverlap (9086 ms) 2024-03-07T03:34:22.187 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/0.ZeroOverlap 2024-03-07T03:34:22.188 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:34:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:34:22 smithi195 bash[59134]: ts=2024-03-07T03:34:22.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:23.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:22 smithi148 bash[79819]: cluster 2024-03-07T03:34:20.887902+0000 mgr.y (mgr.34101) 612 : cluster 0 pgmap v1111: 105 pgs: 105 active+clean; 315 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.4 MiB/s wr, 75 op/s 2024-03-07T03:34:23.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:22 smithi148 bash[82222]: cluster 2024-03-07T03:34:20.887902+0000 mgr.y (mgr.34101) 612 : cluster 0 pgmap v1111: 105 pgs: 105 active+clean; 315 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.4 MiB/s wr, 75 op/s 2024-03-07T03:34:23.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:22 smithi195 bash[66542]: cluster 2024-03-07T03:34:20.887902+0000 mgr.y (mgr.34101) 612 : cluster 0 pgmap v1111: 105 pgs: 105 active+clean; 315 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.4 MiB/s wr, 75 op/s 2024-03-07T03:34:24.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:23 smithi148 bash[79819]: cluster 2024-03-07T03:34:22.678539+0000 mon.a (mon.0) 1497 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2024-03-07T03:34:24.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:23 smithi148 bash[82222]: cluster 2024-03-07T03:34:22.678539+0000 mon.a (mon.0) 1497 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2024-03-07T03:34:24.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:23 smithi195 bash[66542]: cluster 2024-03-07T03:34:22.678539+0000 mon.a (mon.0) 1497 : cluster 0 osdmap e881: 8 total, 8 up, 8 in 2024-03-07T03:34:25.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:24 smithi148 bash[79819]: cluster 2024-03-07T03:34:22.888608+0000 mgr.y (mgr.34101) 613 : cluster 0 pgmap v1113: 105 pgs: 105 active+clean; 315 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.0 MiB/s wr, 77 op/s 2024-03-07T03:34:25.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:24 smithi148 bash[82222]: cluster 2024-03-07T03:34:22.888608+0000 mgr.y (mgr.34101) 613 : cluster 0 pgmap v1113: 105 pgs: 105 active+clean; 315 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.0 MiB/s wr, 77 op/s 2024-03-07T03:34:25.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:24 smithi195 bash[66542]: cluster 2024-03-07T03:34:22.888608+0000 mgr.y (mgr.34101) 613 : cluster 0 pgmap v1113: 105 pgs: 105 active+clean; 315 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.0 MiB/s wr, 77 op/s 2024-03-07T03:34:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:26 smithi148 bash[79819]: cluster 2024-03-07T03:34:24.889801+0000 mgr.y (mgr.34101) 614 : cluster 0 pgmap v1114: 105 pgs: 105 active+clean; 336 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.2 MiB/s wr, 74 op/s 2024-03-07T03:34:27.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:26 smithi148 bash[82222]: cluster 2024-03-07T03:34:24.889801+0000 mgr.y (mgr.34101) 614 : cluster 0 pgmap v1114: 105 pgs: 105 active+clean; 336 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.2 MiB/s wr, 74 op/s 2024-03-07T03:34:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:26 smithi195 bash[66542]: cluster 2024-03-07T03:34:24.889801+0000 mgr.y (mgr.34101) 614 : cluster 0 pgmap v1114: 105 pgs: 105 active+clean; 336 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 6.2 MiB/s wr, 74 op/s 2024-03-07T03:34:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:34:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:34:27] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:34:28.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:27 smithi148 bash[79819]: audit 2024-03-07T03:34:27.329915+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:28.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:27 smithi148 bash[82222]: audit 2024-03-07T03:34:27.329915+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:27 smithi195 bash[66542]: audit 2024-03-07T03:34:27.329915+0000 mon.a (mon.0) 1498 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:29.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:28 smithi148 bash[79819]: cluster 2024-03-07T03:34:26.890751+0000 mgr.y (mgr.34101) 615 : cluster 0 pgmap v1115: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 65 op/s 2024-03-07T03:34:29.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:28 smithi148 bash[82222]: cluster 2024-03-07T03:34:26.890751+0000 mgr.y (mgr.34101) 615 : cluster 0 pgmap v1115: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 65 op/s 2024-03-07T03:34:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:28 smithi195 bash[66542]: cluster 2024-03-07T03:34:26.890751+0000 mgr.y (mgr.34101) 615 : cluster 0 pgmap v1115: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 65 op/s 2024-03-07T03:34:31.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:30 smithi148 bash[82222]: cluster 2024-03-07T03:34:28.891753+0000 mgr.y (mgr.34101) 616 : cluster 0 pgmap v1116: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.7 MiB/s wr, 63 op/s 2024-03-07T03:34:31.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:30 smithi148 bash[79819]: cluster 2024-03-07T03:34:28.891753+0000 mgr.y (mgr.34101) 616 : cluster 0 pgmap v1116: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.7 MiB/s wr, 63 op/s 2024-03-07T03:34:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:30 smithi195 bash[66542]: cluster 2024-03-07T03:34:28.891753+0000 mgr.y (mgr.34101) 616 : cluster 0 pgmap v1116: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.7 MiB/s wr, 63 op/s 2024-03-07T03:34:31.196 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/0.ZeroOverlap (9009 ms) 2024-03-07T03:34:31.715 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/0 (68630 ms total) 2024-03-07T03:34:31.715 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:34:31.715 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/1, where TypeParam = PlaintextUnderLUKS2 2024-03-07T03:34:31.747 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/1.Simple 2024-03-07T03:34:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:34:32 smithi195 bash[59134]: ts=2024-03-07T03:34:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:33.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:32 smithi148 bash[79819]: cluster 2024-03-07T03:34:30.892712+0000 mgr.y (mgr.34101) 617 : cluster 0 pgmap v1117: 105 pgs: 105 active+clean; 344 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.7 MiB/s wr, 59 op/s 2024-03-07T03:34:33.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:32 smithi148 bash[79819]: cluster 2024-03-07T03:34:31.713048+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2024-03-07T03:34:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:32 smithi148 bash[82222]: cluster 2024-03-07T03:34:30.892712+0000 mgr.y (mgr.34101) 617 : cluster 0 pgmap v1117: 105 pgs: 105 active+clean; 344 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.7 MiB/s wr, 59 op/s 2024-03-07T03:34:33.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:32 smithi148 bash[82222]: cluster 2024-03-07T03:34:31.713048+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2024-03-07T03:34:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:32 smithi195 bash[66542]: cluster 2024-03-07T03:34:30.892712+0000 mgr.y (mgr.34101) 617 : cluster 0 pgmap v1117: 105 pgs: 105 active+clean; 344 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.7 MiB/s wr, 59 op/s 2024-03-07T03:34:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:32 smithi195 bash[66542]: cluster 2024-03-07T03:34:31.713048+0000 mon.a (mon.0) 1499 : cluster 0 osdmap e882: 8 total, 8 up, 8 in 2024-03-07T03:34:33.735 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:34:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:33 smithi148 bash[79819]: cluster 2024-03-07T03:34:32.723729+0000 mon.a (mon.0) 1500 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2024-03-07T03:34:34.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:33 smithi148 bash[79819]: audit 2024-03-07T03:34:32.729632+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.148:0/1314905038' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:34.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:33 smithi148 bash[79819]: audit 2024-03-07T03:34:32.731000+0000 mon.a (mon.0) 1501 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:33 smithi148 bash[82222]: cluster 2024-03-07T03:34:32.723729+0000 mon.a (mon.0) 1500 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2024-03-07T03:34:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:33 smithi148 bash[82222]: audit 2024-03-07T03:34:32.729632+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.148:0/1314905038' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:34.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:33 smithi148 bash[82222]: audit 2024-03-07T03:34:32.731000+0000 mon.a (mon.0) 1501 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:34.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:33 smithi195 bash[66542]: cluster 2024-03-07T03:34:32.723729+0000 mon.a (mon.0) 1500 : cluster 0 osdmap e883: 8 total, 8 up, 8 in 2024-03-07T03:34:34.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:33 smithi195 bash[66542]: audit 2024-03-07T03:34:32.729632+0000 mon.c (mon.2) 80 : audit 1 from='client.? 172.21.15.148:0/1314905038' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:34.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:33 smithi195 bash[66542]: audit 2024-03-07T03:34:32.731000+0000 mon.a (mon.0) 1501 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[79819]: cluster 2024-03-07T03:34:32.893376+0000 mgr.y (mgr.34101) 618 : cluster 0 pgmap v1120: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[79819]: cluster 2024-03-07T03:34:33.717670+0000 mon.a (mon.0) 1502 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[79819]: audit 2024-03-07T03:34:33.728561+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[79819]: cluster 2024-03-07T03:34:33.742314+0000 mon.a (mon.0) 1504 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[82222]: cluster 2024-03-07T03:34:32.893376+0000 mgr.y (mgr.34101) 618 : cluster 0 pgmap v1120: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[82222]: cluster 2024-03-07T03:34:33.717670+0000 mon.a (mon.0) 1502 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[82222]: audit 2024-03-07T03:34:33.728561+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:34:35.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:34 smithi148 bash[82222]: cluster 2024-03-07T03:34:33.742314+0000 mon.a (mon.0) 1504 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2024-03-07T03:34:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:34 smithi195 bash[66542]: cluster 2024-03-07T03:34:32.893376+0000 mgr.y (mgr.34101) 618 : cluster 0 pgmap v1120: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:34 smithi195 bash[66542]: cluster 2024-03-07T03:34:33.717670+0000 mon.a (mon.0) 1502 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:34:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:34 smithi195 bash[66542]: audit 2024-03-07T03:34:33.728561+0000 mon.a (mon.0) 1503 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:34:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:34 smithi195 bash[66542]: cluster 2024-03-07T03:34:33.742314+0000 mon.a (mon.0) 1504 : cluster 0 osdmap e884: 8 total, 8 up, 8 in 2024-03-07T03:34:36.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:35 smithi148 bash[79819]: cluster 2024-03-07T03:34:34.739187+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2024-03-07T03:34:36.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:35 smithi148 bash[79819]: cluster 2024-03-07T03:34:34.894074+0000 mgr.y (mgr.34101) 619 : cluster 0 pgmap v1123: 105 pgs: 12 creating+peering, 10 unknown, 83 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:35 smithi148 bash[82222]: cluster 2024-03-07T03:34:34.739187+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2024-03-07T03:34:36.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:35 smithi148 bash[82222]: cluster 2024-03-07T03:34:34.894074+0000 mgr.y (mgr.34101) 619 : cluster 0 pgmap v1123: 105 pgs: 12 creating+peering, 10 unknown, 83 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:36.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:35 smithi195 bash[66542]: cluster 2024-03-07T03:34:34.739187+0000 mon.a (mon.0) 1505 : cluster 0 osdmap e885: 8 total, 8 up, 8 in 2024-03-07T03:34:36.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:35 smithi195 bash[66542]: cluster 2024-03-07T03:34:34.894074+0000 mgr.y (mgr.34101) 619 : cluster 0 pgmap v1123: 105 pgs: 12 creating+peering, 10 unknown, 83 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:34:37.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:36 smithi148 bash[82222]: cluster 2024-03-07T03:34:35.742757+0000 mon.a (mon.0) 1506 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:34:37.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:36 smithi148 bash[82222]: cluster 2024-03-07T03:34:35.758058+0000 mon.a (mon.0) 1507 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2024-03-07T03:34:37.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:36 smithi148 bash[82222]: audit 2024-03-07T03:34:36.535734+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:34:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:36 smithi148 bash[79819]: cluster 2024-03-07T03:34:35.742757+0000 mon.a (mon.0) 1506 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:34:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:36 smithi148 bash[79819]: cluster 2024-03-07T03:34:35.758058+0000 mon.a (mon.0) 1507 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2024-03-07T03:34:37.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:36 smithi148 bash[79819]: audit 2024-03-07T03:34:36.535734+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:34:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:36 smithi195 bash[66542]: cluster 2024-03-07T03:34:35.742757+0000 mon.a (mon.0) 1506 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:34:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:36 smithi195 bash[66542]: cluster 2024-03-07T03:34:35.758058+0000 mon.a (mon.0) 1507 : cluster 0 osdmap e886: 8 total, 8 up, 8 in 2024-03-07T03:34:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:36 smithi195 bash[66542]: audit 2024-03-07T03:34:36.535734+0000 mon.a (mon.0) 1508 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:34:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:34:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:34:37] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:34:38.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:37 smithi148 bash[79819]: cluster 2024-03-07T03:34:36.765007+0000 mon.a (mon.0) 1509 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2024-03-07T03:34:38.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:37 smithi148 bash[79819]: cluster 2024-03-07T03:34:36.894713+0000 mgr.y (mgr.34101) 620 : cluster 0 pgmap v1126: 105 pgs: 12 creating+peering, 93 active+clean; 4.6 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1024 KiB/s wr, 0 op/s 2024-03-07T03:34:38.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:37 smithi148 bash[82222]: cluster 2024-03-07T03:34:36.765007+0000 mon.a (mon.0) 1509 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2024-03-07T03:34:38.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:37 smithi148 bash[82222]: cluster 2024-03-07T03:34:36.894713+0000 mgr.y (mgr.34101) 620 : cluster 0 pgmap v1126: 105 pgs: 12 creating+peering, 93 active+clean; 4.6 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1024 KiB/s wr, 0 op/s 2024-03-07T03:34:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:37 smithi195 bash[66542]: cluster 2024-03-07T03:34:36.765007+0000 mon.a (mon.0) 1509 : cluster 0 osdmap e887: 8 total, 8 up, 8 in 2024-03-07T03:34:38.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:37 smithi195 bash[66542]: cluster 2024-03-07T03:34:36.894713+0000 mgr.y (mgr.34101) 620 : cluster 0 pgmap v1126: 105 pgs: 12 creating+peering, 93 active+clean; 4.6 MiB data, 615 MiB used, 715 GiB / 715 GiB avail; 1024 KiB/s wr, 0 op/s 2024-03-07T03:34:40.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:39 smithi148 bash[79819]: cluster 2024-03-07T03:34:38.895505+0000 mgr.y (mgr.34101) 621 : cluster 0 pgmap v1127: 105 pgs: 105 active+clean; 22 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-03-07T03:34:40.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:39 smithi148 bash[82222]: cluster 2024-03-07T03:34:38.895505+0000 mgr.y (mgr.34101) 621 : cluster 0 pgmap v1127: 105 pgs: 105 active+clean; 22 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-03-07T03:34:40.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:39 smithi195 bash[66542]: cluster 2024-03-07T03:34:38.895505+0000 mgr.y (mgr.34101) 621 : cluster 0 pgmap v1127: 105 pgs: 105 active+clean; 22 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-03-07T03:34:42.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:41 smithi148 bash[79819]: cluster 2024-03-07T03:34:40.896192+0000 mgr.y (mgr.34101) 622 : cluster 0 pgmap v1128: 105 pgs: 105 active+clean; 26 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.1 MiB/s wr, 60 op/s 2024-03-07T03:34:42.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:41 smithi148 bash[82222]: cluster 2024-03-07T03:34:40.896192+0000 mgr.y (mgr.34101) 622 : cluster 0 pgmap v1128: 105 pgs: 105 active+clean; 26 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.1 MiB/s wr, 60 op/s 2024-03-07T03:34:42.303 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:41 smithi195 bash[66542]: cluster 2024-03-07T03:34:40.896192+0000 mgr.y (mgr.34101) 622 : cluster 0 pgmap v1128: 105 pgs: 105 active+clean; 26 MiB data, 603 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.1 MiB/s wr, 60 op/s 2024-03-07T03:34:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:34:42 smithi195 bash[59134]: ts=2024-03-07T03:34:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:43.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[79819]: audit 2024-03-07T03:34:42.329511+0000 mon.a (mon.0) 1510 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:43.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[79819]: cluster 2024-03-07T03:34:42.458015+0000 mon.a (mon.0) 1511 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[79819]: audit 2024-03-07T03:34:42.471307+0000 mon.a (mon.0) 1512 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[79819]: audit 2024-03-07T03:34:42.479620+0000 mon.a (mon.0) 1513 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[79819]: audit 2024-03-07T03:34:42.627083+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[79819]: audit 2024-03-07T03:34:42.638250+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[82222]: audit 2024-03-07T03:34:42.329511+0000 mon.a (mon.0) 1510 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[82222]: cluster 2024-03-07T03:34:42.458015+0000 mon.a (mon.0) 1511 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[82222]: audit 2024-03-07T03:34:42.471307+0000 mon.a (mon.0) 1512 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[82222]: audit 2024-03-07T03:34:42.479620+0000 mon.a (mon.0) 1513 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[82222]: audit 2024-03-07T03:34:42.627083+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:42 smithi148 bash[82222]: audit 2024-03-07T03:34:42.638250+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:42 smithi195 bash[66542]: audit 2024-03-07T03:34:42.329511+0000 mon.a (mon.0) 1510 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:42 smithi195 bash[66542]: cluster 2024-03-07T03:34:42.458015+0000 mon.a (mon.0) 1511 : cluster 0 osdmap e888: 8 total, 8 up, 8 in 2024-03-07T03:34:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:42 smithi195 bash[66542]: audit 2024-03-07T03:34:42.471307+0000 mon.a (mon.0) 1512 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:42 smithi195 bash[66542]: audit 2024-03-07T03:34:42.479620+0000 mon.a (mon.0) 1513 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:42 smithi195 bash[66542]: audit 2024-03-07T03:34:42.627083+0000 mon.a (mon.0) 1514 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:43.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:42 smithi195 bash[66542]: audit 2024-03-07T03:34:42.638250+0000 mon.a (mon.0) 1515 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[79819]: cluster 2024-03-07T03:34:42.896711+0000 mgr.y (mgr.34101) 623 : cluster 0 pgmap v1130: 105 pgs: 105 active+clean; 26 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 3.5 MiB/s wr, 51 op/s 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[79819]: audit 2024-03-07T03:34:43.212439+0000 mon.a (mon.0) 1516 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[79819]: audit 2024-03-07T03:34:43.213516+0000 mon.a (mon.0) 1517 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[79819]: audit 2024-03-07T03:34:43.220534+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[79819]: audit 2024-03-07T03:34:43.551000+0000 mon.a (mon.0) 1519 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[82222]: cluster 2024-03-07T03:34:42.896711+0000 mgr.y (mgr.34101) 623 : cluster 0 pgmap v1130: 105 pgs: 105 active+clean; 26 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 3.5 MiB/s wr, 51 op/s 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[82222]: audit 2024-03-07T03:34:43.212439+0000 mon.a (mon.0) 1516 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:34:44.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[82222]: audit 2024-03-07T03:34:43.213516+0000 mon.a (mon.0) 1517 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:34:44.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[82222]: audit 2024-03-07T03:34:43.220534+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:44.292 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:43 smithi148 bash[82222]: audit 2024-03-07T03:34:43.551000+0000 mon.a (mon.0) 1519 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:44.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:43 smithi195 bash[66542]: cluster 2024-03-07T03:34:42.896711+0000 mgr.y (mgr.34101) 623 : cluster 0 pgmap v1130: 105 pgs: 105 active+clean; 26 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 3.5 MiB/s wr, 51 op/s 2024-03-07T03:34:44.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:43 smithi195 bash[66542]: audit 2024-03-07T03:34:43.212439+0000 mon.a (mon.0) 1516 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:34:44.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:43 smithi195 bash[66542]: audit 2024-03-07T03:34:43.213516+0000 mon.a (mon.0) 1517 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:34:44.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:43 smithi195 bash[66542]: audit 2024-03-07T03:34:43.220534+0000 mon.a (mon.0) 1518 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:44.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:43 smithi195 bash[66542]: audit 2024-03-07T03:34:43.551000+0000 mon.a (mon.0) 1519 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:34:46.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:45 smithi148 bash[79819]: cluster 2024-03-07T03:34:44.897486+0000 mgr.y (mgr.34101) 624 : cluster 0 pgmap v1131: 105 pgs: 105 active+clean; 26 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.6 MiB/s wr, 45 op/s 2024-03-07T03:34:46.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:45 smithi148 bash[82222]: cluster 2024-03-07T03:34:44.897486+0000 mgr.y (mgr.34101) 624 : cluster 0 pgmap v1131: 105 pgs: 105 active+clean; 26 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.6 MiB/s wr, 45 op/s 2024-03-07T03:34:46.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:45 smithi195 bash[66542]: cluster 2024-03-07T03:34:44.897486+0000 mgr.y (mgr.34101) 624 : cluster 0 pgmap v1131: 105 pgs: 105 active+clean; 26 MiB data, 601 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.6 MiB/s wr, 45 op/s 2024-03-07T03:34:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:34:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:34:47] "GET /metrics HTTP/1.1" 200 36723 "" "Prometheus/2.43.0" 2024-03-07T03:34:48.250 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/1.Simple (16503 ms) 2024-03-07T03:34:48.250 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/1.Grow 2024-03-07T03:34:48.250 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:34:48.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:47 smithi148 bash[79819]: cluster 2024-03-07T03:34:46.898129+0000 mgr.y (mgr.34101) 625 : cluster 0 pgmap v1132: 105 pgs: 105 active+clean; 34 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-03-07T03:34:48.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:47 smithi148 bash[82222]: cluster 2024-03-07T03:34:46.898129+0000 mgr.y (mgr.34101) 625 : cluster 0 pgmap v1132: 105 pgs: 105 active+clean; 34 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-03-07T03:34:48.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:47 smithi195 bash[66542]: cluster 2024-03-07T03:34:46.898129+0000 mgr.y (mgr.34101) 625 : cluster 0 pgmap v1132: 105 pgs: 105 active+clean; 34 MiB data, 624 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-03-07T03:34:50.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:49 smithi148 bash[79819]: cluster 2024-03-07T03:34:48.899194+0000 mgr.y (mgr.34101) 626 : cluster 0 pgmap v1133: 105 pgs: 105 active+clean; 75 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.1 MiB/s wr, 41 op/s 2024-03-07T03:34:50.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:49 smithi148 bash[79819]: cluster 2024-03-07T03:34:48.979324+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2024-03-07T03:34:50.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:49 smithi148 bash[82222]: cluster 2024-03-07T03:34:48.899194+0000 mgr.y (mgr.34101) 626 : cluster 0 pgmap v1133: 105 pgs: 105 active+clean; 75 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.1 MiB/s wr, 41 op/s 2024-03-07T03:34:50.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:49 smithi148 bash[82222]: cluster 2024-03-07T03:34:48.979324+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2024-03-07T03:34:50.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:49 smithi195 bash[66542]: cluster 2024-03-07T03:34:48.899194+0000 mgr.y (mgr.34101) 626 : cluster 0 pgmap v1133: 105 pgs: 105 active+clean; 75 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.1 MiB/s wr, 41 op/s 2024-03-07T03:34:50.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:49 smithi195 bash[66542]: cluster 2024-03-07T03:34:48.979324+0000 mon.a (mon.0) 1520 : cluster 0 osdmap e889: 8 total, 8 up, 8 in 2024-03-07T03:34:52.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:51 smithi148 bash[79819]: cluster 2024-03-07T03:34:50.899958+0000 mgr.y (mgr.34101) 627 : cluster 0 pgmap v1135: 105 pgs: 105 active+clean; 92 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-03-07T03:34:52.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:51 smithi148 bash[82222]: cluster 2024-03-07T03:34:50.899958+0000 mgr.y (mgr.34101) 627 : cluster 0 pgmap v1135: 105 pgs: 105 active+clean; 92 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-03-07T03:34:52.305 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:51 smithi195 bash[66542]: cluster 2024-03-07T03:34:50.899958+0000 mgr.y (mgr.34101) 627 : cluster 0 pgmap v1135: 105 pgs: 105 active+clean; 92 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 14 MiB/s rd, 9.2 MiB/s wr, 42 op/s 2024-03-07T03:34:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:34:52 smithi195 bash[59134]: ts=2024-03-07T03:34:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:34:54.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:53 smithi148 bash[79819]: cluster 2024-03-07T03:34:52.900559+0000 mgr.y (mgr.34101) 628 : cluster 0 pgmap v1136: 105 pgs: 105 active+clean; 92 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 41 op/s 2024-03-07T03:34:54.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:53 smithi148 bash[82222]: cluster 2024-03-07T03:34:52.900559+0000 mgr.y (mgr.34101) 628 : cluster 0 pgmap v1136: 105 pgs: 105 active+clean; 92 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 41 op/s 2024-03-07T03:34:54.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:53 smithi195 bash[66542]: cluster 2024-03-07T03:34:52.900559+0000 mgr.y (mgr.34101) 628 : cluster 0 pgmap v1136: 105 pgs: 105 active+clean; 92 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 41 op/s 2024-03-07T03:34:56.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:55 smithi148 bash[79819]: cluster 2024-03-07T03:34:54.901771+0000 mgr.y (mgr.34101) 629 : cluster 0 pgmap v1137: 105 pgs: 105 active+clean; 92 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 66 op/s 2024-03-07T03:34:56.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:55 smithi148 bash[82222]: cluster 2024-03-07T03:34:54.901771+0000 mgr.y (mgr.34101) 629 : cluster 0 pgmap v1137: 105 pgs: 105 active+clean; 92 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 66 op/s 2024-03-07T03:34:56.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:55 smithi195 bash[66542]: cluster 2024-03-07T03:34:54.901771+0000 mgr.y (mgr.34101) 629 : cluster 0 pgmap v1137: 105 pgs: 105 active+clean; 92 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 66 op/s 2024-03-07T03:34:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:34:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:34:57] "GET /metrics HTTP/1.1" 200 36723 "" "Prometheus/2.43.0" 2024-03-07T03:34:58.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:57 smithi148 bash[79819]: cluster 2024-03-07T03:34:56.902485+0000 mgr.y (mgr.34101) 630 : cluster 0 pgmap v1138: 105 pgs: 105 active+clean; 92 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.4 MiB/s wr, 65 op/s 2024-03-07T03:34:58.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:57 smithi148 bash[79819]: audit 2024-03-07T03:34:57.329875+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:58.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:57 smithi148 bash[82222]: cluster 2024-03-07T03:34:56.902485+0000 mgr.y (mgr.34101) 630 : cluster 0 pgmap v1138: 105 pgs: 105 active+clean; 92 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.4 MiB/s wr, 65 op/s 2024-03-07T03:34:58.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:57 smithi148 bash[82222]: audit 2024-03-07T03:34:57.329875+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:34:58.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:57 smithi195 bash[66542]: cluster 2024-03-07T03:34:56.902485+0000 mgr.y (mgr.34101) 630 : cluster 0 pgmap v1138: 105 pgs: 105 active+clean; 92 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.4 MiB/s wr, 65 op/s 2024-03-07T03:34:58.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:57 smithi195 bash[66542]: audit 2024-03-07T03:34:57.329875+0000 mon.a (mon.0) 1521 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:00.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:34:59 smithi148 bash[79819]: cluster 2024-03-07T03:34:58.903286+0000 mgr.y (mgr.34101) 631 : cluster 0 pgmap v1139: 105 pgs: 105 active+clean; 108 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:35:00.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:34:59 smithi148 bash[82222]: cluster 2024-03-07T03:34:58.903286+0000 mgr.y (mgr.34101) 631 : cluster 0 pgmap v1139: 105 pgs: 105 active+clean; 108 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:35:00.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:34:59 smithi195 bash[66542]: cluster 2024-03-07T03:34:58.903286+0000 mgr.y (mgr.34101) 631 : cluster 0 pgmap v1139: 105 pgs: 105 active+clean; 108 MiB data, 828 MiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:35:00.712 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/1.Grow (12463 ms) 2024-03-07T03:35:00.713 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/1.Shrink 2024-03-07T03:35:00.713 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:35:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:02 smithi148 bash[79819]: cluster 2024-03-07T03:35:00.904120+0000 mgr.y (mgr.34101) 632 : cluster 0 pgmap v1140: 105 pgs: 105 active+clean; 125 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 5.9 MiB/s wr, 37 op/s 2024-03-07T03:35:02.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:02 smithi148 bash[79819]: cluster 2024-03-07T03:35:01.011403+0000 mon.a (mon.0) 1522 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2024-03-07T03:35:02.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:02 smithi148 bash[82222]: cluster 2024-03-07T03:35:00.904120+0000 mgr.y (mgr.34101) 632 : cluster 0 pgmap v1140: 105 pgs: 105 active+clean; 125 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 5.9 MiB/s wr, 37 op/s 2024-03-07T03:35:02.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:02 smithi148 bash[82222]: cluster 2024-03-07T03:35:01.011403+0000 mon.a (mon.0) 1522 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2024-03-07T03:35:02.305 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:02 smithi195 bash[66542]: cluster 2024-03-07T03:35:00.904120+0000 mgr.y (mgr.34101) 632 : cluster 0 pgmap v1140: 105 pgs: 105 active+clean; 125 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 5.9 MiB/s wr, 37 op/s 2024-03-07T03:35:02.305 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:02 smithi195 bash[66542]: cluster 2024-03-07T03:35:01.011403+0000 mon.a (mon.0) 1522 : cluster 0 osdmap e890: 8 total, 8 up, 8 in 2024-03-07T03:35:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:35:02 smithi195 bash[59134]: ts=2024-03-07T03:35:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:04.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:04 smithi148 bash[79819]: cluster 2024-03-07T03:35:02.904771+0000 mgr.y (mgr.34101) 633 : cluster 0 pgmap v1142: 105 pgs: 105 active+clean; 125 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 4.5 MiB/s wr, 46 op/s 2024-03-07T03:35:04.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:04 smithi148 bash[82222]: cluster 2024-03-07T03:35:02.904771+0000 mgr.y (mgr.34101) 633 : cluster 0 pgmap v1142: 105 pgs: 105 active+clean; 125 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 4.5 MiB/s wr, 46 op/s 2024-03-07T03:35:04.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:04 smithi195 bash[66542]: cluster 2024-03-07T03:35:02.904771+0000 mgr.y (mgr.34101) 633 : cluster 0 pgmap v1142: 105 pgs: 105 active+clean; 125 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 4.5 MiB/s wr, 46 op/s 2024-03-07T03:35:06.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:06 smithi148 bash[79819]: cluster 2024-03-07T03:35:04.905469+0000 mgr.y (mgr.34101) 634 : cluster 0 pgmap v1143: 105 pgs: 105 active+clean; 154 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.8 MiB/s wr, 67 op/s 2024-03-07T03:35:06.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:06 smithi148 bash[82222]: cluster 2024-03-07T03:35:04.905469+0000 mgr.y (mgr.34101) 634 : cluster 0 pgmap v1143: 105 pgs: 105 active+clean; 154 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.8 MiB/s wr, 67 op/s 2024-03-07T03:35:06.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:06 smithi195 bash[66542]: cluster 2024-03-07T03:35:04.905469+0000 mgr.y (mgr.34101) 634 : cluster 0 pgmap v1143: 105 pgs: 105 active+clean; 154 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.8 MiB/s wr, 67 op/s 2024-03-07T03:35:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:35:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:35:07] "GET /metrics HTTP/1.1" 200 36728 "" "Prometheus/2.43.0" 2024-03-07T03:35:08.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:08 smithi148 bash[79819]: cluster 2024-03-07T03:35:06.906114+0000 mgr.y (mgr.34101) 635 : cluster 0 pgmap v1144: 105 pgs: 105 active+clean; 158 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:35:08.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:08 smithi148 bash[82222]: cluster 2024-03-07T03:35:06.906114+0000 mgr.y (mgr.34101) 635 : cluster 0 pgmap v1144: 105 pgs: 105 active+clean; 158 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:35:08.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:08 smithi195 bash[66542]: cluster 2024-03-07T03:35:06.906114+0000 mgr.y (mgr.34101) 635 : cluster 0 pgmap v1144: 105 pgs: 105 active+clean; 158 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:35:10.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:10 smithi148 bash[79819]: cluster 2024-03-07T03:35:08.906823+0000 mgr.y (mgr.34101) 636 : cluster 0 pgmap v1145: 105 pgs: 105 active+clean; 158 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 6.2 MiB/s wr, 62 op/s 2024-03-07T03:35:10.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:10 smithi148 bash[82222]: cluster 2024-03-07T03:35:08.906823+0000 mgr.y (mgr.34101) 636 : cluster 0 pgmap v1145: 105 pgs: 105 active+clean; 158 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 6.2 MiB/s wr, 62 op/s 2024-03-07T03:35:10.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:10 smithi195 bash[66542]: cluster 2024-03-07T03:35:08.906823+0000 mgr.y (mgr.34101) 636 : cluster 0 pgmap v1145: 105 pgs: 105 active+clean; 158 MiB data, 1010 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 6.2 MiB/s wr, 62 op/s 2024-03-07T03:35:12.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:12 smithi148 bash[79819]: cluster 2024-03-07T03:35:10.907466+0000 mgr.y (mgr.34101) 637 : cluster 0 pgmap v1146: 105 pgs: 105 active+clean; 162 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 4.1 MiB/s wr, 63 op/s 2024-03-07T03:35:12.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:12 smithi148 bash[82222]: cluster 2024-03-07T03:35:10.907466+0000 mgr.y (mgr.34101) 637 : cluster 0 pgmap v1146: 105 pgs: 105 active+clean; 162 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 4.1 MiB/s wr, 63 op/s 2024-03-07T03:35:12.304 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:12 smithi195 bash[66542]: cluster 2024-03-07T03:35:10.907466+0000 mgr.y (mgr.34101) 637 : cluster 0 pgmap v1146: 105 pgs: 105 active+clean; 162 MiB data, 1021 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 4.1 MiB/s wr, 63 op/s 2024-03-07T03:35:12.513 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/1.Shrink (11801 ms) 2024-03-07T03:35:12.514 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/1.ShrinkToOne 2024-03-07T03:35:12.514 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:35:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:35:12 smithi195 bash[59134]: ts=2024-03-07T03:35:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:13.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:13 smithi148 bash[82222]: audit 2024-03-07T03:35:12.330620+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:13.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:13 smithi148 bash[79819]: audit 2024-03-07T03:35:12.330620+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:13.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:13 smithi195 bash[66542]: audit 2024-03-07T03:35:12.330620+0000 mon.a (mon.0) 1523 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:14.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:14 smithi148 bash[82222]: cluster 2024-03-07T03:35:12.908258+0000 mgr.y (mgr.34101) 638 : cluster 0 pgmap v1147: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.5 MiB/s wr, 54 op/s 2024-03-07T03:35:14.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:14 smithi148 bash[82222]: cluster 2024-03-07T03:35:13.037840+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2024-03-07T03:35:14.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:14 smithi148 bash[79819]: cluster 2024-03-07T03:35:12.908258+0000 mgr.y (mgr.34101) 638 : cluster 0 pgmap v1147: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.5 MiB/s wr, 54 op/s 2024-03-07T03:35:14.291 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:14 smithi148 bash[79819]: cluster 2024-03-07T03:35:13.037840+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2024-03-07T03:35:14.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:14 smithi195 bash[66542]: cluster 2024-03-07T03:35:12.908258+0000 mgr.y (mgr.34101) 638 : cluster 0 pgmap v1147: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.5 MiB/s wr, 54 op/s 2024-03-07T03:35:14.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:14 smithi195 bash[66542]: cluster 2024-03-07T03:35:13.037840+0000 mon.a (mon.0) 1524 : cluster 0 osdmap e891: 8 total, 8 up, 8 in 2024-03-07T03:35:16.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:16 smithi148 bash[79819]: cluster 2024-03-07T03:35:14.909163+0000 mgr.y (mgr.34101) 639 : cluster 0 pgmap v1149: 105 pgs: 105 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:35:16.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:16 smithi148 bash[82222]: cluster 2024-03-07T03:35:14.909163+0000 mgr.y (mgr.34101) 639 : cluster 0 pgmap v1149: 105 pgs: 105 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:35:16.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:16 smithi195 bash[66542]: cluster 2024-03-07T03:35:14.909163+0000 mgr.y (mgr.34101) 639 : cluster 0 pgmap v1149: 105 pgs: 105 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 7.0 MiB/s wr, 63 op/s 2024-03-07T03:35:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:35:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:35:17] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.43.0" 2024-03-07T03:35:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:18 smithi148 bash[79819]: cluster 2024-03-07T03:35:16.909794+0000 mgr.y (mgr.34101) 640 : cluster 0 pgmap v1150: 105 pgs: 105 active+clean; 224 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 69 op/s 2024-03-07T03:35:18.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:18 smithi148 bash[82222]: cluster 2024-03-07T03:35:16.909794+0000 mgr.y (mgr.34101) 640 : cluster 0 pgmap v1150: 105 pgs: 105 active+clean; 224 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 69 op/s 2024-03-07T03:35:18.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:18 smithi195 bash[66542]: cluster 2024-03-07T03:35:16.909794+0000 mgr.y (mgr.34101) 640 : cluster 0 pgmap v1150: 105 pgs: 105 active+clean; 224 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 69 op/s 2024-03-07T03:35:20.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:20 smithi148 bash[79819]: cluster 2024-03-07T03:35:18.910525+0000 mgr.y (mgr.34101) 641 : cluster 0 pgmap v1151: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 64 op/s 2024-03-07T03:35:20.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:20 smithi148 bash[82222]: cluster 2024-03-07T03:35:18.910525+0000 mgr.y (mgr.34101) 641 : cluster 0 pgmap v1151: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 64 op/s 2024-03-07T03:35:20.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:20 smithi195 bash[66542]: cluster 2024-03-07T03:35:18.910525+0000 mgr.y (mgr.34101) 641 : cluster 0 pgmap v1151: 105 pgs: 105 active+clean; 224 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 64 op/s 2024-03-07T03:35:22.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:22 smithi148 bash[79819]: cluster 2024-03-07T03:35:20.911166+0000 mgr.y (mgr.34101) 642 : cluster 0 pgmap v1152: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 62 op/s 2024-03-07T03:35:22.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:22 smithi148 bash[82222]: cluster 2024-03-07T03:35:20.911166+0000 mgr.y (mgr.34101) 642 : cluster 0 pgmap v1152: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 62 op/s 2024-03-07T03:35:22.305 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:22 smithi195 bash[66542]: cluster 2024-03-07T03:35:20.911166+0000 mgr.y (mgr.34101) 642 : cluster 0 pgmap v1152: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 62 op/s 2024-03-07T03:35:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:35:22 smithi195 bash[59134]: ts=2024-03-07T03:35:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:24.042 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/1.ShrinkToOne (11528 ms) 2024-03-07T03:35:24.042 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/1.ShrinkToOneAfterSnapshot 2024-03-07T03:35:24.042 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:35:24.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:24 smithi148 bash[79819]: cluster 2024-03-07T03:35:22.911727+0000 mgr.y (mgr.34101) 643 : cluster 0 pgmap v1153: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 64 op/s 2024-03-07T03:35:24.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:24 smithi148 bash[82222]: cluster 2024-03-07T03:35:22.911727+0000 mgr.y (mgr.34101) 643 : cluster 0 pgmap v1153: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 64 op/s 2024-03-07T03:35:24.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:24 smithi195 bash[66542]: cluster 2024-03-07T03:35:22.911727+0000 mgr.y (mgr.34101) 643 : cluster 0 pgmap v1153: 105 pgs: 105 active+clean; 228 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 64 op/s 2024-03-07T03:35:26.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:26 smithi195 bash[66542]: cluster 2024-03-07T03:35:24.912935+0000 mgr.y (mgr.34101) 644 : cluster 0 pgmap v1154: 105 pgs: 105 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.3 MiB/s wr, 68 op/s 2024-03-07T03:35:26.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:26 smithi195 bash[66542]: cluster 2024-03-07T03:35:25.068819+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2024-03-07T03:35:26.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:26 smithi148 bash[79819]: cluster 2024-03-07T03:35:24.912935+0000 mgr.y (mgr.34101) 644 : cluster 0 pgmap v1154: 105 pgs: 105 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.3 MiB/s wr, 68 op/s 2024-03-07T03:35:26.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:26 smithi148 bash[79819]: cluster 2024-03-07T03:35:25.068819+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2024-03-07T03:35:26.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:26 smithi148 bash[82222]: cluster 2024-03-07T03:35:24.912935+0000 mgr.y (mgr.34101) 644 : cluster 0 pgmap v1154: 105 pgs: 105 active+clean; 244 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.3 MiB/s wr, 68 op/s 2024-03-07T03:35:26.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:26 smithi148 bash[82222]: cluster 2024-03-07T03:35:25.068819+0000 mon.a (mon.0) 1525 : cluster 0 osdmap e892: 8 total, 8 up, 8 in 2024-03-07T03:35:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:35:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:35:27] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.43.0" 2024-03-07T03:35:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:28 smithi195 bash[66542]: cluster 2024-03-07T03:35:26.913663+0000 mgr.y (mgr.34101) 645 : cluster 0 pgmap v1156: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:35:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:28 smithi195 bash[66542]: audit 2024-03-07T03:35:27.330846+0000 mon.a (mon.0) 1526 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:28.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:28 smithi148 bash[79819]: cluster 2024-03-07T03:35:26.913663+0000 mgr.y (mgr.34101) 645 : cluster 0 pgmap v1156: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:35:28.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:28 smithi148 bash[79819]: audit 2024-03-07T03:35:27.330846+0000 mon.a (mon.0) 1526 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:28.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:28 smithi148 bash[82222]: cluster 2024-03-07T03:35:26.913663+0000 mgr.y (mgr.34101) 645 : cluster 0 pgmap v1156: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.8 MiB/s wr, 28 op/s 2024-03-07T03:35:28.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:28 smithi148 bash[82222]: audit 2024-03-07T03:35:27.330846+0000 mon.a (mon.0) 1526 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:30.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:30 smithi195 bash[66542]: cluster 2024-03-07T03:35:28.914703+0000 mgr.y (mgr.34101) 646 : cluster 0 pgmap v1157: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-03-07T03:35:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:30 smithi148 bash[79819]: cluster 2024-03-07T03:35:28.914703+0000 mgr.y (mgr.34101) 646 : cluster 0 pgmap v1157: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-03-07T03:35:30.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:30 smithi148 bash[82222]: cluster 2024-03-07T03:35:28.914703+0000 mgr.y (mgr.34101) 646 : cluster 0 pgmap v1157: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-03-07T03:35:32.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:32 smithi195 bash[66542]: cluster 2024-03-07T03:35:30.915397+0000 mgr.y (mgr.34101) 647 : cluster 0 pgmap v1158: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:35:32.315 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:35:32 smithi195 bash[59134]: ts=2024-03-07T03:35:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:32.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:32 smithi148 bash[79819]: cluster 2024-03-07T03:35:30.915397+0000 mgr.y (mgr.34101) 647 : cluster 0 pgmap v1158: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:35:32.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:32 smithi148 bash[82222]: cluster 2024-03-07T03:35:30.915397+0000 mgr.y (mgr.34101) 647 : cluster 0 pgmap v1158: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 59 op/s 2024-03-07T03:35:34.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:34 smithi195 bash[66542]: cluster 2024-03-07T03:35:32.915965+0000 mgr.y (mgr.34101) 648 : cluster 0 pgmap v1159: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 57 op/s 2024-03-07T03:35:34.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:34 smithi148 bash[79819]: cluster 2024-03-07T03:35:32.915965+0000 mgr.y (mgr.34101) 648 : cluster 0 pgmap v1159: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 57 op/s 2024-03-07T03:35:34.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:34 smithi148 bash[82222]: cluster 2024-03-07T03:35:32.915965+0000 mgr.y (mgr.34101) 648 : cluster 0 pgmap v1159: 105 pgs: 105 active+clean; 269 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.1 MiB/s wr, 57 op/s 2024-03-07T03:35:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:36 smithi148 bash[79819]: cluster 2024-03-07T03:35:34.917152+0000 mgr.y (mgr.34101) 649 : cluster 0 pgmap v1160: 105 pgs: 105 active+clean; 281 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 506 KiB/s rd, 4.5 MiB/s wr, 51 op/s 2024-03-07T03:35:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:36 smithi148 bash[79819]: cluster 2024-03-07T03:35:35.092047+0000 mon.a (mon.0) 1527 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2024-03-07T03:35:36.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:36 smithi148 bash[82222]: cluster 2024-03-07T03:35:34.917152+0000 mgr.y (mgr.34101) 649 : cluster 0 pgmap v1160: 105 pgs: 105 active+clean; 281 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 506 KiB/s rd, 4.5 MiB/s wr, 51 op/s 2024-03-07T03:35:36.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:36 smithi148 bash[82222]: cluster 2024-03-07T03:35:35.092047+0000 mon.a (mon.0) 1527 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2024-03-07T03:35:36.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:36 smithi195 bash[66542]: cluster 2024-03-07T03:35:34.917152+0000 mgr.y (mgr.34101) 649 : cluster 0 pgmap v1160: 105 pgs: 105 active+clean; 281 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 506 KiB/s rd, 4.5 MiB/s wr, 51 op/s 2024-03-07T03:35:36.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:36 smithi195 bash[66542]: cluster 2024-03-07T03:35:35.092047+0000 mon.a (mon.0) 1527 : cluster 0 osdmap e893: 8 total, 8 up, 8 in 2024-03-07T03:35:37.183 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/1.ShrinkToOneAfterSnapshot (13141 ms) 2024-03-07T03:35:37.183 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/1.MinOverlap 2024-03-07T03:35:37.183 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:35:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:35:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:35:37] "GET /metrics HTTP/1.1" 200 36729 "" "Prometheus/2.43.0" 2024-03-07T03:35:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:38 smithi148 bash[79819]: cluster 2024-03-07T03:35:36.917821+0000 mgr.y (mgr.34101) 650 : cluster 0 pgmap v1162: 105 pgs: 105 active+clean; 306 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 470 KiB/s rd, 5.4 MiB/s wr, 44 op/s 2024-03-07T03:35:38.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:38 smithi148 bash[82222]: cluster 2024-03-07T03:35:36.917821+0000 mgr.y (mgr.34101) 650 : cluster 0 pgmap v1162: 105 pgs: 105 active+clean; 306 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 470 KiB/s rd, 5.4 MiB/s wr, 44 op/s 2024-03-07T03:35:38.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:38 smithi195 bash[66542]: cluster 2024-03-07T03:35:36.917821+0000 mgr.y (mgr.34101) 650 : cluster 0 pgmap v1162: 105 pgs: 105 active+clean; 306 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 470 KiB/s rd, 5.4 MiB/s wr, 44 op/s 2024-03-07T03:35:39.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:39 smithi148 bash[79819]: cluster 2024-03-07T03:35:38.105928+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2024-03-07T03:35:39.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:39 smithi148 bash[82222]: cluster 2024-03-07T03:35:38.105928+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2024-03-07T03:35:39.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:39 smithi195 bash[66542]: cluster 2024-03-07T03:35:38.105928+0000 mon.a (mon.0) 1528 : cluster 0 osdmap e894: 8 total, 8 up, 8 in 2024-03-07T03:35:40.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:40 smithi148 bash[79819]: cluster 2024-03-07T03:35:38.918766+0000 mgr.y (mgr.34101) 651 : cluster 0 pgmap v1164: 105 pgs: 105 active+clean; 358 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 11 MiB/s wr, 74 op/s 2024-03-07T03:35:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:40 smithi148 bash[82222]: cluster 2024-03-07T03:35:38.918766+0000 mgr.y (mgr.34101) 651 : cluster 0 pgmap v1164: 105 pgs: 105 active+clean; 358 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 11 MiB/s wr, 74 op/s 2024-03-07T03:35:40.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:40 smithi195 bash[66542]: cluster 2024-03-07T03:35:38.918766+0000 mgr.y (mgr.34101) 651 : cluster 0 pgmap v1164: 105 pgs: 105 active+clean; 358 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 11 MiB/s wr, 74 op/s 2024-03-07T03:35:42.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:42 smithi148 bash[79819]: cluster 2024-03-07T03:35:40.919485+0000 mgr.y (mgr.34101) 652 : cluster 0 pgmap v1165: 105 pgs: 105 active+clean; 375 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 13 MiB/s wr, 96 op/s 2024-03-07T03:35:42.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:42 smithi148 bash[82222]: cluster 2024-03-07T03:35:40.919485+0000 mgr.y (mgr.34101) 652 : cluster 0 pgmap v1165: 105 pgs: 105 active+clean; 375 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 13 MiB/s wr, 96 op/s 2024-03-07T03:35:42.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:42 smithi195 bash[66542]: cluster 2024-03-07T03:35:40.919485+0000 mgr.y (mgr.34101) 652 : cluster 0 pgmap v1165: 105 pgs: 105 active+clean; 375 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 13 MiB/s wr, 96 op/s 2024-03-07T03:35:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:35:42 smithi195 bash[59134]: ts=2024-03-07T03:35:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:43 smithi148 bash[79819]: audit 2024-03-07T03:35:42.330964+0000 mon.a (mon.0) 1529 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:43.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:43 smithi148 bash[82222]: audit 2024-03-07T03:35:42.330964+0000 mon.a (mon.0) 1529 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:43.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:43 smithi195 bash[66542]: audit 2024-03-07T03:35:42.330964+0000 mon.a (mon.0) 1529 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:44 smithi148 bash[79819]: cluster 2024-03-07T03:35:42.920065+0000 mgr.y (mgr.34101) 653 : cluster 0 pgmap v1166: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 12 MiB/s wr, 84 op/s 2024-03-07T03:35:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:44 smithi148 bash[79819]: audit 2024-03-07T03:35:43.300715+0000 mon.a (mon.0) 1530 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:35:44.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:44 smithi148 bash[82222]: cluster 2024-03-07T03:35:42.920065+0000 mgr.y (mgr.34101) 653 : cluster 0 pgmap v1166: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 12 MiB/s wr, 84 op/s 2024-03-07T03:35:44.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:44 smithi148 bash[82222]: audit 2024-03-07T03:35:43.300715+0000 mon.a (mon.0) 1530 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:35:44.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:44 smithi195 bash[66542]: cluster 2024-03-07T03:35:42.920065+0000 mgr.y (mgr.34101) 653 : cluster 0 pgmap v1166: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 12 MiB/s wr, 84 op/s 2024-03-07T03:35:44.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:44 smithi195 bash[66542]: audit 2024-03-07T03:35:43.300715+0000 mon.a (mon.0) 1530 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:35:46.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:46 smithi148 bash[79819]: cluster 2024-03-07T03:35:44.920793+0000 mgr.y (mgr.34101) 654 : cluster 0 pgmap v1167: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.6 MiB/s wr, 71 op/s 2024-03-07T03:35:46.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:46 smithi148 bash[82222]: cluster 2024-03-07T03:35:44.920793+0000 mgr.y (mgr.34101) 654 : cluster 0 pgmap v1167: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.6 MiB/s wr, 71 op/s 2024-03-07T03:35:46.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:46 smithi195 bash[66542]: cluster 2024-03-07T03:35:44.920793+0000 mgr.y (mgr.34101) 654 : cluster 0 pgmap v1167: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.6 MiB/s wr, 71 op/s 2024-03-07T03:35:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:35:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:35:47] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:35:48.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:48 smithi148 bash[79819]: cluster 2024-03-07T03:35:46.921420+0000 mgr.y (mgr.34101) 655 : cluster 0 pgmap v1168: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.3 MiB/s wr, 68 op/s 2024-03-07T03:35:48.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:48 smithi148 bash[82222]: cluster 2024-03-07T03:35:46.921420+0000 mgr.y (mgr.34101) 655 : cluster 0 pgmap v1168: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.3 MiB/s wr, 68 op/s 2024-03-07T03:35:48.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:48 smithi195 bash[66542]: cluster 2024-03-07T03:35:46.921420+0000 mgr.y (mgr.34101) 655 : cluster 0 pgmap v1168: 105 pgs: 105 active+clean; 379 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.3 MiB/s wr, 68 op/s 2024-03-07T03:35:50.120 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/1.MinOverlap (12938 ms) 2024-03-07T03:35:50.120 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/1.ZeroOverlap 2024-03-07T03:35:50.120 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: cluster 2024-03-07T03:35:48.922040+0000 mgr.y (mgr.34101) 656 : cluster 0 pgmap v1169: 105 pgs: 105 active+clean; 399 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 10 MiB/s wr, 86 op/s 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: audit 2024-03-07T03:35:49.291560+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: audit 2024-03-07T03:35:49.296828+0000 mon.a (mon.0) 1532 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: audit 2024-03-07T03:35:49.344482+0000 mon.a (mon.0) 1533 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: audit 2024-03-07T03:35:49.354531+0000 mon.a (mon.0) 1534 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: audit 2024-03-07T03:35:49.989141+0000 mon.a (mon.0) 1535 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: audit 2024-03-07T03:35:49.990589+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:35:50.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[79819]: audit 2024-03-07T03:35:49.999327+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: cluster 2024-03-07T03:35:48.922040+0000 mgr.y (mgr.34101) 656 : cluster 0 pgmap v1169: 105 pgs: 105 active+clean; 399 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 10 MiB/s wr, 86 op/s 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: audit 2024-03-07T03:35:49.291560+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: audit 2024-03-07T03:35:49.296828+0000 mon.a (mon.0) 1532 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: audit 2024-03-07T03:35:49.344482+0000 mon.a (mon.0) 1533 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: audit 2024-03-07T03:35:49.354531+0000 mon.a (mon.0) 1534 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: audit 2024-03-07T03:35:49.989141+0000 mon.a (mon.0) 1535 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: audit 2024-03-07T03:35:49.990589+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:35:50.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:50 smithi148 bash[82222]: audit 2024-03-07T03:35:49.999327+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: cluster 2024-03-07T03:35:48.922040+0000 mgr.y (mgr.34101) 656 : cluster 0 pgmap v1169: 105 pgs: 105 active+clean; 399 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.8 MiB/s rd, 10 MiB/s wr, 86 op/s 2024-03-07T03:35:50.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: audit 2024-03-07T03:35:49.291560+0000 mon.a (mon.0) 1531 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: audit 2024-03-07T03:35:49.296828+0000 mon.a (mon.0) 1532 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: audit 2024-03-07T03:35:49.344482+0000 mon.a (mon.0) 1533 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: audit 2024-03-07T03:35:49.354531+0000 mon.a (mon.0) 1534 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:50.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: audit 2024-03-07T03:35:49.989141+0000 mon.a (mon.0) 1535 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:35:50.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: audit 2024-03-07T03:35:49.990589+0000 mon.a (mon.0) 1536 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:35:50.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:50 smithi195 bash[66542]: audit 2024-03-07T03:35:49.999327+0000 mon.a (mon.0) 1537 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:35:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:52 smithi195 bash[66542]: cluster 2024-03-07T03:35:50.922950+0000 mgr.y (mgr.34101) 657 : cluster 0 pgmap v1170: 105 pgs: 105 active+clean; 399 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-03-07T03:35:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:52 smithi195 bash[66542]: cluster 2024-03-07T03:35:51.308446+0000 mon.a (mon.0) 1538 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2024-03-07T03:35:52.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:35:52 smithi195 bash[59134]: ts=2024-03-07T03:35:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:35:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:52 smithi148 bash[79819]: cluster 2024-03-07T03:35:50.922950+0000 mgr.y (mgr.34101) 657 : cluster 0 pgmap v1170: 105 pgs: 105 active+clean; 399 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-03-07T03:35:52.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:52 smithi148 bash[79819]: cluster 2024-03-07T03:35:51.308446+0000 mon.a (mon.0) 1538 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2024-03-07T03:35:52.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:52 smithi148 bash[82222]: cluster 2024-03-07T03:35:50.922950+0000 mgr.y (mgr.34101) 657 : cluster 0 pgmap v1170: 105 pgs: 105 active+clean; 399 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 5.1 MiB/s wr, 39 op/s 2024-03-07T03:35:52.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:52 smithi148 bash[82222]: cluster 2024-03-07T03:35:51.308446+0000 mon.a (mon.0) 1538 : cluster 0 osdmap e895: 8 total, 8 up, 8 in 2024-03-07T03:35:54.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:54 smithi195 bash[66542]: cluster 2024-03-07T03:35:52.923601+0000 mgr.y (mgr.34101) 658 : cluster 0 pgmap v1172: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.4 MiB/s wr, 28 op/s 2024-03-07T03:35:54.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:54 smithi148 bash[79819]: cluster 2024-03-07T03:35:52.923601+0000 mgr.y (mgr.34101) 658 : cluster 0 pgmap v1172: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.4 MiB/s wr, 28 op/s 2024-03-07T03:35:54.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:54 smithi148 bash[82222]: cluster 2024-03-07T03:35:52.923601+0000 mgr.y (mgr.34101) 658 : cluster 0 pgmap v1172: 105 pgs: 105 active+clean; 403 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.4 MiB/s wr, 28 op/s 2024-03-07T03:35:56.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:56 smithi195 bash[66542]: cluster 2024-03-07T03:35:54.924327+0000 mgr.y (mgr.34101) 659 : cluster 0 pgmap v1173: 105 pgs: 105 active+clean; 424 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 42 op/s 2024-03-07T03:35:56.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:56 smithi148 bash[79819]: cluster 2024-03-07T03:35:54.924327+0000 mgr.y (mgr.34101) 659 : cluster 0 pgmap v1173: 105 pgs: 105 active+clean; 424 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 42 op/s 2024-03-07T03:35:56.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:56 smithi148 bash[82222]: cluster 2024-03-07T03:35:54.924327+0000 mgr.y (mgr.34101) 659 : cluster 0 pgmap v1173: 105 pgs: 105 active+clean; 424 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 42 op/s 2024-03-07T03:35:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:35:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:35:57] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:35:58.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:58 smithi195 bash[66542]: cluster 2024-03-07T03:35:56.924946+0000 mgr.y (mgr.34101) 660 : cluster 0 pgmap v1174: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-03-07T03:35:58.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:35:58 smithi195 bash[66542]: audit 2024-03-07T03:35:57.331326+0000 mon.a (mon.0) 1539 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:58.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:58 smithi148 bash[79819]: cluster 2024-03-07T03:35:56.924946+0000 mgr.y (mgr.34101) 660 : cluster 0 pgmap v1174: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-03-07T03:35:58.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:35:58 smithi148 bash[79819]: audit 2024-03-07T03:35:57.331326+0000 mon.a (mon.0) 1539 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:35:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:58 smithi148 bash[82222]: cluster 2024-03-07T03:35:56.924946+0000 mgr.y (mgr.34101) 660 : cluster 0 pgmap v1174: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-03-07T03:35:58.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:35:58 smithi148 bash[82222]: audit 2024-03-07T03:35:57.331326+0000 mon.a (mon.0) 1539 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:00.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:00 smithi195 bash[66542]: cluster 2024-03-07T03:35:58.925669+0000 mgr.y (mgr.34101) 661 : cluster 0 pgmap v1175: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 875 KiB/s rd, 2.5 MiB/s wr, 37 op/s 2024-03-07T03:36:00.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:00 smithi148 bash[79819]: cluster 2024-03-07T03:35:58.925669+0000 mgr.y (mgr.34101) 661 : cluster 0 pgmap v1175: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 875 KiB/s rd, 2.5 MiB/s wr, 37 op/s 2024-03-07T03:36:00.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:00 smithi148 bash[82222]: cluster 2024-03-07T03:35:58.925669+0000 mgr.y (mgr.34101) 661 : cluster 0 pgmap v1175: 105 pgs: 105 active+clean; 424 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 875 KiB/s rd, 2.5 MiB/s wr, 37 op/s 2024-03-07T03:36:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:02 smithi195 bash[66542]: cluster 2024-03-07T03:36:00.926317+0000 mgr.y (mgr.34101) 662 : cluster 0 pgmap v1176: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 908 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:36:02.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:36:02 smithi195 bash[59134]: ts=2024-03-07T03:36:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:02.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:02 smithi148 bash[79819]: cluster 2024-03-07T03:36:00.926317+0000 mgr.y (mgr.34101) 662 : cluster 0 pgmap v1176: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 908 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:36:02.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:02 smithi148 bash[82222]: cluster 2024-03-07T03:36:00.926317+0000 mgr.y (mgr.34101) 662 : cluster 0 pgmap v1176: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 908 KiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:36:03.399 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/1.ZeroOverlap (13279 ms) 2024-03-07T03:36:04.333 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/1 (91653 ms total) 2024-03-07T03:36:04.333 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:36:04.333 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/2, where TypeParam = UnformattedLUKS1 2024-03-07T03:36:04.384 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/2.Simple 2024-03-07T03:36:04.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:04 smithi148 bash[79819]: cluster 2024-03-07T03:36:02.926890+0000 mgr.y (mgr.34101) 663 : cluster 0 pgmap v1177: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 782 KiB/s rd, 2.8 MiB/s wr, 38 op/s 2024-03-07T03:36:04.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:04 smithi148 bash[82222]: cluster 2024-03-07T03:36:02.926890+0000 mgr.y (mgr.34101) 663 : cluster 0 pgmap v1177: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 782 KiB/s rd, 2.8 MiB/s wr, 38 op/s 2024-03-07T03:36:04.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:04 smithi195 bash[66542]: cluster 2024-03-07T03:36:02.926890+0000 mgr.y (mgr.34101) 663 : cluster 0 pgmap v1177: 105 pgs: 105 active+clean; 432 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 782 KiB/s rd, 2.8 MiB/s wr, 38 op/s 2024-03-07T03:36:05.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:05 smithi148 bash[79819]: cluster 2024-03-07T03:36:04.337007+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2024-03-07T03:36:05.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:05 smithi148 bash[82222]: cluster 2024-03-07T03:36:04.337007+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2024-03-07T03:36:05.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:05 smithi195 bash[66542]: cluster 2024-03-07T03:36:04.337007+0000 mon.a (mon.0) 1540 : cluster 0 osdmap e896: 8 total, 8 up, 8 in 2024-03-07T03:36:06.368 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:36:06.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[82222]: cluster 2024-03-07T03:36:04.927467+0000 mgr.y (mgr.34101) 664 : cluster 0 pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:36:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[82222]: cluster 2024-03-07T03:36:05.360692+0000 mon.a (mon.0) 1541 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2024-03-07T03:36:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[82222]: audit 2024-03-07T03:36:05.370293+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.148:0/4141545480' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:36:06.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[82222]: audit 2024-03-07T03:36:05.371606+0000 mon.a (mon.0) 1542 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:36:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[79819]: cluster 2024-03-07T03:36:04.927467+0000 mgr.y (mgr.34101) 664 : cluster 0 pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:36:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[79819]: cluster 2024-03-07T03:36:05.360692+0000 mon.a (mon.0) 1541 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2024-03-07T03:36:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[79819]: audit 2024-03-07T03:36:05.370293+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.148:0/4141545480' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:36:06.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:06 smithi148 bash[79819]: audit 2024-03-07T03:36:05.371606+0000 mon.a (mon.0) 1542 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:36:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:06 smithi195 bash[66542]: cluster 2024-03-07T03:36:04.927467+0000 mgr.y (mgr.34101) 664 : cluster 0 pgmap v1179: 73 pgs: 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:36:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:06 smithi195 bash[66542]: cluster 2024-03-07T03:36:05.360692+0000 mon.a (mon.0) 1541 : cluster 0 osdmap e897: 8 total, 8 up, 8 in 2024-03-07T03:36:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:06 smithi195 bash[66542]: audit 2024-03-07T03:36:05.370293+0000 mon.c (mon.2) 81 : audit 1 from='client.? 172.21.15.148:0/4141545480' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:36:06.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:06 smithi195 bash[66542]: audit 2024-03-07T03:36:05.371606+0000 mon.a (mon.0) 1542 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:36:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:36:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:36:07] "GET /metrics HTTP/1.1" 200 36735 "" "Prometheus/2.43.0" 2024-03-07T03:36:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:07 smithi148 bash[79819]: audit 2024-03-07T03:36:06.363986+0000 mon.a (mon.0) 1543 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:36:07.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:07 smithi148 bash[79819]: cluster 2024-03-07T03:36:06.370908+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2024-03-07T03:36:07.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:07 smithi148 bash[82222]: audit 2024-03-07T03:36:06.363986+0000 mon.a (mon.0) 1543 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:36:07.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:07 smithi148 bash[82222]: cluster 2024-03-07T03:36:06.370908+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2024-03-07T03:36:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:07 smithi195 bash[66542]: audit 2024-03-07T03:36:06.363986+0000 mon.a (mon.0) 1543 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:36:07.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:07 smithi195 bash[66542]: cluster 2024-03-07T03:36:06.370908+0000 mon.a (mon.0) 1544 : cluster 0 osdmap e898: 8 total, 8 up, 8 in 2024-03-07T03:36:08.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:08 smithi148 bash[79819]: cluster 2024-03-07T03:36:06.928123+0000 mgr.y (mgr.34101) 665 : cluster 0 pgmap v1182: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:36:08.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:08 smithi148 bash[79819]: cluster 2024-03-07T03:36:07.384980+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2024-03-07T03:36:08.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:08 smithi148 bash[82222]: cluster 2024-03-07T03:36:06.928123+0000 mgr.y (mgr.34101) 665 : cluster 0 pgmap v1182: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:36:08.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:08 smithi148 bash[82222]: cluster 2024-03-07T03:36:07.384980+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2024-03-07T03:36:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:08 smithi195 bash[66542]: cluster 2024-03-07T03:36:06.928123+0000 mgr.y (mgr.34101) 665 : cluster 0 pgmap v1182: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:36:08.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:08 smithi195 bash[66542]: cluster 2024-03-07T03:36:07.384980+0000 mon.a (mon.0) 1545 : cluster 0 osdmap e899: 8 total, 8 up, 8 in 2024-03-07T03:36:09.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:09 smithi148 bash[79819]: cluster 2024-03-07T03:36:08.390511+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2024-03-07T03:36:09.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:09 smithi148 bash[82222]: cluster 2024-03-07T03:36:08.390511+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2024-03-07T03:36:09.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:09 smithi195 bash[66542]: cluster 2024-03-07T03:36:08.390511+0000 mon.a (mon.0) 1546 : cluster 0 osdmap e900: 8 total, 8 up, 8 in 2024-03-07T03:36:10.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:10 smithi148 bash[82222]: cluster 2024-03-07T03:36:08.928898+0000 mgr.y (mgr.34101) 666 : cluster 0 pgmap v1185: 105 pgs: 5 creating+peering, 8 unknown, 92 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:36:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:10 smithi148 bash[79819]: cluster 2024-03-07T03:36:08.928898+0000 mgr.y (mgr.34101) 666 : cluster 0 pgmap v1185: 105 pgs: 5 creating+peering, 8 unknown, 92 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:36:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:10 smithi195 bash[66542]: cluster 2024-03-07T03:36:08.928898+0000 mgr.y (mgr.34101) 666 : cluster 0 pgmap v1185: 105 pgs: 5 creating+peering, 8 unknown, 92 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:36:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:12 smithi195 bash[66542]: cluster 2024-03-07T03:36:10.929868+0000 mgr.y (mgr.34101) 667 : cluster 0 pgmap v1186: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 183 B/s rd, 367 B/s wr, 0 op/s 2024-03-07T03:36:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:12 smithi195 bash[66542]: audit 2024-03-07T03:36:12.332374+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:12.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:36:12 smithi195 bash[59134]: ts=2024-03-07T03:36:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:12 smithi148 bash[82222]: cluster 2024-03-07T03:36:10.929868+0000 mgr.y (mgr.34101) 667 : cluster 0 pgmap v1186: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 183 B/s rd, 367 B/s wr, 0 op/s 2024-03-07T03:36:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:12 smithi148 bash[82222]: audit 2024-03-07T03:36:12.332374+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:12 smithi148 bash[79819]: cluster 2024-03-07T03:36:10.929868+0000 mgr.y (mgr.34101) 667 : cluster 0 pgmap v1186: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 183 B/s rd, 367 B/s wr, 0 op/s 2024-03-07T03:36:12.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:12 smithi148 bash[79819]: audit 2024-03-07T03:36:12.332374+0000 mon.a (mon.0) 1547 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:13.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:13 smithi148 bash[82222]: cluster 2024-03-07T03:36:12.406331+0000 mon.a (mon.0) 1548 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2024-03-07T03:36:13.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:13 smithi148 bash[79819]: cluster 2024-03-07T03:36:12.406331+0000 mon.a (mon.0) 1548 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2024-03-07T03:36:13.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:13 smithi195 bash[66542]: cluster 2024-03-07T03:36:12.406331+0000 mon.a (mon.0) 1548 : cluster 0 osdmap e901: 8 total, 8 up, 8 in 2024-03-07T03:36:14.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:14 smithi148 bash[82222]: cluster 2024-03-07T03:36:12.930526+0000 mgr.y (mgr.34101) 668 : cluster 0 pgmap v1188: 105 pgs: 105 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 852 B/s wr, 2 op/s 2024-03-07T03:36:14.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:14 smithi148 bash[79819]: cluster 2024-03-07T03:36:12.930526+0000 mgr.y (mgr.34101) 668 : cluster 0 pgmap v1188: 105 pgs: 105 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 852 B/s wr, 2 op/s 2024-03-07T03:36:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:14 smithi195 bash[66542]: cluster 2024-03-07T03:36:12.930526+0000 mgr.y (mgr.34101) 668 : cluster 0 pgmap v1188: 105 pgs: 105 active+clean; 582 KiB data, 533 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 852 B/s wr, 2 op/s 2024-03-07T03:36:16.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:16 smithi148 bash[82222]: cluster 2024-03-07T03:36:14.931656+0000 mgr.y (mgr.34101) 669 : cluster 0 pgmap v1189: 105 pgs: 105 active+clean; 18 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 2.3 MiB/s wr, 34 op/s 2024-03-07T03:36:16.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:16 smithi148 bash[79819]: cluster 2024-03-07T03:36:14.931656+0000 mgr.y (mgr.34101) 669 : cluster 0 pgmap v1189: 105 pgs: 105 active+clean; 18 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 2.3 MiB/s wr, 34 op/s 2024-03-07T03:36:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:16 smithi195 bash[66542]: cluster 2024-03-07T03:36:14.931656+0000 mgr.y (mgr.34101) 669 : cluster 0 pgmap v1189: 105 pgs: 105 active+clean; 18 MiB data, 591 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s rd, 2.3 MiB/s wr, 34 op/s 2024-03-07T03:36:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:36:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:36:17] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2024-03-07T03:36:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:18 smithi148 bash[79819]: cluster 2024-03-07T03:36:16.932846+0000 mgr.y (mgr.34101) 670 : cluster 0 pgmap v1190: 105 pgs: 105 active+clean; 30 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 3.4 MiB/s wr, 52 op/s 2024-03-07T03:36:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:18 smithi148 bash[79819]: cluster 2024-03-07T03:36:17.482021+0000 mon.a (mon.0) 1549 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2024-03-07T03:36:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:18 smithi148 bash[82222]: cluster 2024-03-07T03:36:16.932846+0000 mgr.y (mgr.34101) 670 : cluster 0 pgmap v1190: 105 pgs: 105 active+clean; 30 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 3.4 MiB/s wr, 52 op/s 2024-03-07T03:36:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:18 smithi148 bash[82222]: cluster 2024-03-07T03:36:17.482021+0000 mon.a (mon.0) 1549 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2024-03-07T03:36:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:18 smithi195 bash[66542]: cluster 2024-03-07T03:36:16.932846+0000 mgr.y (mgr.34101) 670 : cluster 0 pgmap v1190: 105 pgs: 105 active+clean; 30 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s rd, 3.4 MiB/s wr, 52 op/s 2024-03-07T03:36:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:18 smithi195 bash[66542]: cluster 2024-03-07T03:36:17.482021+0000 mon.a (mon.0) 1549 : cluster 0 osdmap e902: 8 total, 8 up, 8 in 2024-03-07T03:36:19.247 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/2.Simple (14863 ms) 2024-03-07T03:36:19.247 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/2.Grow 2024-03-07T03:36:19.247 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:36:20.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:19 smithi148 bash[79819]: audit 2024-03-07T03:36:18.587386+0000 mon.a (mon.0) 1550 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:20.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:19 smithi148 bash[82222]: audit 2024-03-07T03:36:18.587386+0000 mon.a (mon.0) 1550 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:20.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:19 smithi195 bash[66542]: audit 2024-03-07T03:36:18.587386+0000 mon.a (mon.0) 1550 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:21.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:20 smithi148 bash[82222]: cluster 2024-03-07T03:36:18.933435+0000 mgr.y (mgr.34101) 671 : cluster 0 pgmap v1192: 105 pgs: 105 active+clean; 46 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 5.4 MiB/s rd, 5.6 MiB/s wr, 58 op/s 2024-03-07T03:36:21.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:20 smithi148 bash[79819]: cluster 2024-03-07T03:36:18.933435+0000 mgr.y (mgr.34101) 671 : cluster 0 pgmap v1192: 105 pgs: 105 active+clean; 46 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 5.4 MiB/s rd, 5.6 MiB/s wr, 58 op/s 2024-03-07T03:36:21.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:20 smithi195 bash[66542]: cluster 2024-03-07T03:36:18.933435+0000 mgr.y (mgr.34101) 671 : cluster 0 pgmap v1192: 105 pgs: 105 active+clean; 46 MiB data, 661 MiB used, 715 GiB / 715 GiB avail; 5.4 MiB/s rd, 5.6 MiB/s wr, 58 op/s 2024-03-07T03:36:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:36:22 smithi195 bash[59134]: ts=2024-03-07T03:36:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:23.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:22 smithi148 bash[79819]: cluster 2024-03-07T03:36:20.934310+0000 mgr.y (mgr.34101) 672 : cluster 0 pgmap v1193: 105 pgs: 105 active+clean; 51 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 8.1 MiB/s rd, 5.9 MiB/s wr, 63 op/s 2024-03-07T03:36:23.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:22 smithi148 bash[82222]: cluster 2024-03-07T03:36:20.934310+0000 mgr.y (mgr.34101) 672 : cluster 0 pgmap v1193: 105 pgs: 105 active+clean; 51 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 8.1 MiB/s rd, 5.9 MiB/s wr, 63 op/s 2024-03-07T03:36:23.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:22 smithi195 bash[66542]: cluster 2024-03-07T03:36:20.934310+0000 mgr.y (mgr.34101) 672 : cluster 0 pgmap v1193: 105 pgs: 105 active+clean; 51 MiB data, 691 MiB used, 715 GiB / 715 GiB avail; 8.1 MiB/s rd, 5.9 MiB/s wr, 63 op/s 2024-03-07T03:36:25.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:24 smithi148 bash[79819]: cluster 2024-03-07T03:36:22.935096+0000 mgr.y (mgr.34101) 673 : cluster 0 pgmap v1194: 105 pgs: 105 active+clean; 59 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 9.3 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-03-07T03:36:25.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:24 smithi148 bash[79819]: cluster 2024-03-07T03:36:23.612443+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2024-03-07T03:36:25.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:24 smithi148 bash[82222]: cluster 2024-03-07T03:36:22.935096+0000 mgr.y (mgr.34101) 673 : cluster 0 pgmap v1194: 105 pgs: 105 active+clean; 59 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 9.3 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-03-07T03:36:25.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:24 smithi148 bash[82222]: cluster 2024-03-07T03:36:23.612443+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2024-03-07T03:36:25.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:24 smithi195 bash[66542]: cluster 2024-03-07T03:36:22.935096+0000 mgr.y (mgr.34101) 673 : cluster 0 pgmap v1194: 105 pgs: 105 active+clean; 59 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 9.3 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-03-07T03:36:25.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:24 smithi195 bash[66542]: cluster 2024-03-07T03:36:23.612443+0000 mon.a (mon.0) 1551 : cluster 0 osdmap e903: 8 total, 8 up, 8 in 2024-03-07T03:36:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:26 smithi148 bash[79819]: cluster 2024-03-07T03:36:24.935977+0000 mgr.y (mgr.34101) 674 : cluster 0 pgmap v1196: 105 pgs: 105 active+clean; 75 MiB data, 739 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 5.6 MiB/s wr, 38 op/s 2024-03-07T03:36:27.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:26 smithi148 bash[82222]: cluster 2024-03-07T03:36:24.935977+0000 mgr.y (mgr.34101) 674 : cluster 0 pgmap v1196: 105 pgs: 105 active+clean; 75 MiB data, 739 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 5.6 MiB/s wr, 38 op/s 2024-03-07T03:36:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:26 smithi195 bash[66542]: cluster 2024-03-07T03:36:24.935977+0000 mgr.y (mgr.34101) 674 : cluster 0 pgmap v1196: 105 pgs: 105 active+clean; 75 MiB data, 739 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 5.6 MiB/s wr, 38 op/s 2024-03-07T03:36:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:36:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:36:27] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2024-03-07T03:36:28.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:27 smithi148 bash[79819]: audit 2024-03-07T03:36:27.331875+0000 mon.a (mon.0) 1552 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:28.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:27 smithi148 bash[82222]: audit 2024-03-07T03:36:27.331875+0000 mon.a (mon.0) 1552 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:27 smithi195 bash[66542]: audit 2024-03-07T03:36:27.331875+0000 mon.a (mon.0) 1552 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:29.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:28 smithi148 bash[79819]: cluster 2024-03-07T03:36:26.936927+0000 mgr.y (mgr.34101) 675 : cluster 0 pgmap v1197: 105 pgs: 105 active+clean; 88 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 64 op/s 2024-03-07T03:36:29.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:28 smithi148 bash[82222]: cluster 2024-03-07T03:36:26.936927+0000 mgr.y (mgr.34101) 675 : cluster 0 pgmap v1197: 105 pgs: 105 active+clean; 88 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 64 op/s 2024-03-07T03:36:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:28 smithi195 bash[66542]: cluster 2024-03-07T03:36:26.936927+0000 mgr.y (mgr.34101) 675 : cluster 0 pgmap v1197: 105 pgs: 105 active+clean; 88 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 64 op/s 2024-03-07T03:36:30.437 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/2.Grow (11190 ms) 2024-03-07T03:36:30.438 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/2.Shrink 2024-03-07T03:36:30.438 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:36:31.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:30 smithi148 bash[79819]: cluster 2024-03-07T03:36:28.937572+0000 mgr.y (mgr.34101) 676 : cluster 0 pgmap v1198: 105 pgs: 105 active+clean; 104 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 70 op/s 2024-03-07T03:36:31.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:30 smithi148 bash[82222]: cluster 2024-03-07T03:36:28.937572+0000 mgr.y (mgr.34101) 676 : cluster 0 pgmap v1198: 105 pgs: 105 active+clean; 104 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 70 op/s 2024-03-07T03:36:31.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:30 smithi195 bash[66542]: cluster 2024-03-07T03:36:28.937572+0000 mgr.y (mgr.34101) 676 : cluster 0 pgmap v1198: 105 pgs: 105 active+clean; 104 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 70 op/s 2024-03-07T03:36:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:36:32 smithi195 bash[59134]: ts=2024-03-07T03:36:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:33.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:32 smithi148 bash[79819]: cluster 2024-03-07T03:36:30.938375+0000 mgr.y (mgr.34101) 677 : cluster 0 pgmap v1199: 105 pgs: 105 active+clean; 117 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 6.6 MiB/s wr, 70 op/s 2024-03-07T03:36:33.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:32 smithi148 bash[82222]: cluster 2024-03-07T03:36:30.938375+0000 mgr.y (mgr.34101) 677 : cluster 0 pgmap v1199: 105 pgs: 105 active+clean; 117 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 6.6 MiB/s wr, 70 op/s 2024-03-07T03:36:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:32 smithi195 bash[66542]: cluster 2024-03-07T03:36:30.938375+0000 mgr.y (mgr.34101) 677 : cluster 0 pgmap v1199: 105 pgs: 105 active+clean; 117 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 6.6 MiB/s wr, 70 op/s 2024-03-07T03:36:35.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:34 smithi148 bash[79819]: cluster 2024-03-07T03:36:32.939122+0000 mgr.y (mgr.34101) 678 : cluster 0 pgmap v1200: 105 pgs: 105 active+clean; 117 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.8 MiB/s wr, 61 op/s 2024-03-07T03:36:35.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:34 smithi148 bash[82222]: cluster 2024-03-07T03:36:32.939122+0000 mgr.y (mgr.34101) 678 : cluster 0 pgmap v1200: 105 pgs: 105 active+clean; 117 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.8 MiB/s wr, 61 op/s 2024-03-07T03:36:35.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:34 smithi195 bash[66542]: cluster 2024-03-07T03:36:32.939122+0000 mgr.y (mgr.34101) 678 : cluster 0 pgmap v1200: 105 pgs: 105 active+clean; 117 MiB data, 859 MiB used, 714 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.8 MiB/s wr, 61 op/s 2024-03-07T03:36:36.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:35 smithi148 bash[79819]: cluster 2024-03-07T03:36:34.664252+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2024-03-07T03:36:36.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:35 smithi148 bash[82222]: cluster 2024-03-07T03:36:34.664252+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2024-03-07T03:36:36.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:35 smithi195 bash[66542]: cluster 2024-03-07T03:36:34.664252+0000 mon.a (mon.0) 1553 : cluster 0 osdmap e904: 8 total, 8 up, 8 in 2024-03-07T03:36:37.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:36 smithi148 bash[79819]: cluster 2024-03-07T03:36:34.939952+0000 mgr.y (mgr.34101) 679 : cluster 0 pgmap v1202: 105 pgs: 105 active+clean; 117 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 57 op/s 2024-03-07T03:36:37.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:36 smithi148 bash[82222]: cluster 2024-03-07T03:36:34.939952+0000 mgr.y (mgr.34101) 679 : cluster 0 pgmap v1202: 105 pgs: 105 active+clean; 117 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 57 op/s 2024-03-07T03:36:37.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:36 smithi195 bash[66542]: cluster 2024-03-07T03:36:34.939952+0000 mgr.y (mgr.34101) 679 : cluster 0 pgmap v1202: 105 pgs: 105 active+clean; 117 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.2 MiB/s wr, 57 op/s 2024-03-07T03:36:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:36:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:36:37] "GET /metrics HTTP/1.1" 200 36725 "" "Prometheus/2.43.0" 2024-03-07T03:36:39.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:38 smithi148 bash[79819]: cluster 2024-03-07T03:36:36.940597+0000 mgr.y (mgr.34101) 680 : cluster 0 pgmap v1203: 105 pgs: 105 active+clean; 122 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 45 op/s 2024-03-07T03:36:39.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:38 smithi148 bash[82222]: cluster 2024-03-07T03:36:36.940597+0000 mgr.y (mgr.34101) 680 : cluster 0 pgmap v1203: 105 pgs: 105 active+clean; 122 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 45 op/s 2024-03-07T03:36:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:38 smithi195 bash[66542]: cluster 2024-03-07T03:36:36.940597+0000 mgr.y (mgr.34101) 680 : cluster 0 pgmap v1203: 105 pgs: 105 active+clean; 122 MiB data, 908 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.4 MiB/s wr, 45 op/s 2024-03-07T03:36:41.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:40 smithi148 bash[79819]: cluster 2024-03-07T03:36:38.941583+0000 mgr.y (mgr.34101) 681 : cluster 0 pgmap v1204: 105 pgs: 105 active+clean; 138 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 3.4 MiB/s wr, 55 op/s 2024-03-07T03:36:41.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:40 smithi148 bash[82222]: cluster 2024-03-07T03:36:38.941583+0000 mgr.y (mgr.34101) 681 : cluster 0 pgmap v1204: 105 pgs: 105 active+clean; 138 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 3.4 MiB/s wr, 55 op/s 2024-03-07T03:36:41.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:40 smithi195 bash[66542]: cluster 2024-03-07T03:36:38.941583+0000 mgr.y (mgr.34101) 681 : cluster 0 pgmap v1204: 105 pgs: 105 active+clean; 138 MiB data, 954 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 3.4 MiB/s wr, 55 op/s 2024-03-07T03:36:41.642 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/2.Shrink (11204 ms) 2024-03-07T03:36:41.642 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/2.ShrinkToOne 2024-03-07T03:36:41.642 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:36:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:36:42 smithi195 bash[59134]: ts=2024-03-07T03:36:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:42 smithi148 bash[79819]: cluster 2024-03-07T03:36:40.942479+0000 mgr.y (mgr.34101) 682 : cluster 0 pgmap v1205: 105 pgs: 105 active+clean; 150 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 3.3 MiB/s wr, 47 op/s 2024-03-07T03:36:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:42 smithi148 bash[79819]: audit 2024-03-07T03:36:42.332808+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:42 smithi148 bash[82222]: cluster 2024-03-07T03:36:40.942479+0000 mgr.y (mgr.34101) 682 : cluster 0 pgmap v1205: 105 pgs: 105 active+clean; 150 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 3.3 MiB/s wr, 47 op/s 2024-03-07T03:36:43.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:42 smithi148 bash[82222]: audit 2024-03-07T03:36:42.332808+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:42 smithi195 bash[66542]: cluster 2024-03-07T03:36:40.942479+0000 mgr.y (mgr.34101) 682 : cluster 0 pgmap v1205: 105 pgs: 105 active+clean; 150 MiB data, 985 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 3.3 MiB/s wr, 47 op/s 2024-03-07T03:36:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:42 smithi195 bash[66542]: audit 2024-03-07T03:36:42.332808+0000 mon.a (mon.0) 1554 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:45.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:44 smithi148 bash[79819]: cluster 2024-03-07T03:36:42.943227+0000 mgr.y (mgr.34101) 683 : cluster 0 pgmap v1206: 105 pgs: 105 active+clean; 150 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 3.3 MiB/s wr, 52 op/s 2024-03-07T03:36:45.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:44 smithi148 bash[82222]: cluster 2024-03-07T03:36:42.943227+0000 mgr.y (mgr.34101) 683 : cluster 0 pgmap v1206: 105 pgs: 105 active+clean; 150 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 3.3 MiB/s wr, 52 op/s 2024-03-07T03:36:45.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:44 smithi195 bash[66542]: cluster 2024-03-07T03:36:42.943227+0000 mgr.y (mgr.34101) 683 : cluster 0 pgmap v1206: 105 pgs: 105 active+clean; 150 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 3.9 MiB/s rd, 3.3 MiB/s wr, 52 op/s 2024-03-07T03:36:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:46 smithi148 bash[79819]: cluster 2024-03-07T03:36:44.944422+0000 mgr.y (mgr.34101) 684 : cluster 0 pgmap v1207: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.6 MiB/s wr, 62 op/s 2024-03-07T03:36:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:46 smithi148 bash[79819]: cluster 2024-03-07T03:36:45.701604+0000 mon.a (mon.0) 1555 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2024-03-07T03:36:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:46 smithi148 bash[82222]: cluster 2024-03-07T03:36:44.944422+0000 mgr.y (mgr.34101) 684 : cluster 0 pgmap v1207: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.6 MiB/s wr, 62 op/s 2024-03-07T03:36:47.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:46 smithi148 bash[82222]: cluster 2024-03-07T03:36:45.701604+0000 mon.a (mon.0) 1555 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2024-03-07T03:36:47.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:46 smithi195 bash[66542]: cluster 2024-03-07T03:36:44.944422+0000 mgr.y (mgr.34101) 684 : cluster 0 pgmap v1207: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.6 MiB/s wr, 62 op/s 2024-03-07T03:36:47.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:46 smithi195 bash[66542]: cluster 2024-03-07T03:36:45.701604+0000 mon.a (mon.0) 1555 : cluster 0 osdmap e905: 8 total, 8 up, 8 in 2024-03-07T03:36:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:36:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:36:47] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.43.0" 2024-03-07T03:36:49.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:48 smithi148 bash[82222]: cluster 2024-03-07T03:36:46.945228+0000 mgr.y (mgr.34101) 685 : cluster 0 pgmap v1209: 105 pgs: 105 active+clean; 179 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.7 MiB/s wr, 55 op/s 2024-03-07T03:36:49.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:48 smithi148 bash[79819]: cluster 2024-03-07T03:36:46.945228+0000 mgr.y (mgr.34101) 685 : cluster 0 pgmap v1209: 105 pgs: 105 active+clean; 179 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.7 MiB/s wr, 55 op/s 2024-03-07T03:36:49.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:48 smithi195 bash[66542]: cluster 2024-03-07T03:36:46.945228+0000 mgr.y (mgr.34101) 685 : cluster 0 pgmap v1209: 105 pgs: 105 active+clean; 179 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.7 MiB/s wr, 55 op/s 2024-03-07T03:36:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:50 smithi148 bash[79819]: cluster 2024-03-07T03:36:48.946212+0000 mgr.y (mgr.34101) 686 : cluster 0 pgmap v1210: 105 pgs: 105 active+clean; 196 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 63 op/s 2024-03-07T03:36:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:50 smithi148 bash[79819]: audit 2024-03-07T03:36:50.064497+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:36:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:50 smithi148 bash[82222]: cluster 2024-03-07T03:36:48.946212+0000 mgr.y (mgr.34101) 686 : cluster 0 pgmap v1210: 105 pgs: 105 active+clean; 196 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 63 op/s 2024-03-07T03:36:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:50 smithi148 bash[82222]: audit 2024-03-07T03:36:50.064497+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:36:51.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:50 smithi195 bash[66542]: cluster 2024-03-07T03:36:48.946212+0000 mgr.y (mgr.34101) 686 : cluster 0 pgmap v1210: 105 pgs: 105 active+clean; 196 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 63 op/s 2024-03-07T03:36:51.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:50 smithi195 bash[66542]: audit 2024-03-07T03:36:50.064497+0000 mon.a (mon.0) 1556 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:36:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:36:52 smithi195 bash[59134]: ts=2024-03-07T03:36:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:36:52.587 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/2.ShrinkToOne (10946 ms) 2024-03-07T03:36:52.587 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/2.ShrinkToOneAfterSnapshot 2024-03-07T03:36:52.587 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:36:53.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:52 smithi148 bash[79819]: cluster 2024-03-07T03:36:50.947095+0000 mgr.y (mgr.34101) 687 : cluster 0 pgmap v1211: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 64 op/s 2024-03-07T03:36:53.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:52 smithi148 bash[82222]: cluster 2024-03-07T03:36:50.947095+0000 mgr.y (mgr.34101) 687 : cluster 0 pgmap v1211: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 64 op/s 2024-03-07T03:36:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:52 smithi195 bash[66542]: cluster 2024-03-07T03:36:50.947095+0000 mgr.y (mgr.34101) 687 : cluster 0 pgmap v1211: 105 pgs: 105 active+clean; 204 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.4 MiB/s wr, 64 op/s 2024-03-07T03:36:55.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:54 smithi148 bash[82222]: cluster 2024-03-07T03:36:52.947602+0000 mgr.y (mgr.34101) 688 : cluster 0 pgmap v1212: 105 pgs: 105 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-03-07T03:36:55.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:54 smithi148 bash[79819]: cluster 2024-03-07T03:36:52.947602+0000 mgr.y (mgr.34101) 688 : cluster 0 pgmap v1212: 105 pgs: 105 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-03-07T03:36:55.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:54 smithi195 bash[66542]: cluster 2024-03-07T03:36:52.947602+0000 mgr.y (mgr.34101) 688 : cluster 0 pgmap v1212: 105 pgs: 105 active+clean; 208 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.6 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-03-07T03:36:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[79819]: cluster 2024-03-07T03:36:54.948244+0000 mgr.y (mgr.34101) 689 : cluster 0 pgmap v1213: 105 pgs: 105 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 54 op/s 2024-03-07T03:36:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[79819]: audit 2024-03-07T03:36:55.852203+0000 mon.a (mon.0) 1557 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[79819]: audit 2024-03-07T03:36:55.858040+0000 mon.a (mon.0) 1558 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[79819]: audit 2024-03-07T03:36:56.152340+0000 mon.a (mon.0) 1559 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[79819]: audit 2024-03-07T03:36:56.163567+0000 mon.a (mon.0) 1560 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[82222]: cluster 2024-03-07T03:36:54.948244+0000 mgr.y (mgr.34101) 689 : cluster 0 pgmap v1213: 105 pgs: 105 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 54 op/s 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[82222]: audit 2024-03-07T03:36:55.852203+0000 mon.a (mon.0) 1557 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[82222]: audit 2024-03-07T03:36:55.858040+0000 mon.a (mon.0) 1558 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[82222]: audit 2024-03-07T03:36:56.152340+0000 mon.a (mon.0) 1559 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:56 smithi148 bash[82222]: audit 2024-03-07T03:36:56.163567+0000 mon.a (mon.0) 1560 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:56 smithi195 bash[66542]: cluster 2024-03-07T03:36:54.948244+0000 mgr.y (mgr.34101) 689 : cluster 0 pgmap v1213: 105 pgs: 105 active+clean; 212 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 54 op/s 2024-03-07T03:36:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:56 smithi195 bash[66542]: audit 2024-03-07T03:36:55.852203+0000 mon.a (mon.0) 1557 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:56 smithi195 bash[66542]: audit 2024-03-07T03:36:55.858040+0000 mon.a (mon.0) 1558 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:56 smithi195 bash[66542]: audit 2024-03-07T03:36:56.152340+0000 mon.a (mon.0) 1559 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:56 smithi195 bash[66542]: audit 2024-03-07T03:36:56.163567+0000 mon.a (mon.0) 1560 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:36:57] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.43.0" 2024-03-07T03:36:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[79819]: audit 2024-03-07T03:36:56.795731+0000 mon.a (mon.0) 1561 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:36:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[79819]: audit 2024-03-07T03:36:56.797657+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:36:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[79819]: audit 2024-03-07T03:36:56.808343+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[79819]: cluster 2024-03-07T03:36:57.171129+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2024-03-07T03:36:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[79819]: audit 2024-03-07T03:36:57.333861+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[82222]: audit 2024-03-07T03:36:56.795731+0000 mon.a (mon.0) 1561 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:36:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[82222]: audit 2024-03-07T03:36:56.797657+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:36:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[82222]: audit 2024-03-07T03:36:56.808343+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[82222]: cluster 2024-03-07T03:36:57.171129+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2024-03-07T03:36:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:57 smithi148 bash[82222]: audit 2024-03-07T03:36:57.333861+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:57 smithi195 bash[66542]: audit 2024-03-07T03:36:56.795731+0000 mon.a (mon.0) 1561 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:36:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:57 smithi195 bash[66542]: audit 2024-03-07T03:36:56.797657+0000 mon.a (mon.0) 1562 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:36:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:57 smithi195 bash[66542]: audit 2024-03-07T03:36:56.808343+0000 mon.a (mon.0) 1563 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:36:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:57 smithi195 bash[66542]: cluster 2024-03-07T03:36:57.171129+0000 mon.a (mon.0) 1564 : cluster 0 osdmap e906: 8 total, 8 up, 8 in 2024-03-07T03:36:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:57 smithi195 bash[66542]: audit 2024-03-07T03:36:57.333861+0000 mon.a (mon.0) 1565 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:36:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:36:58 smithi148 bash[79819]: cluster 2024-03-07T03:36:56.949120+0000 mgr.y (mgr.34101) 690 : cluster 0 pgmap v1214: 105 pgs: 105 active+clean; 213 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 842 KiB/s rd, 3.7 MiB/s wr, 51 op/s 2024-03-07T03:36:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:36:58 smithi148 bash[82222]: cluster 2024-03-07T03:36:56.949120+0000 mgr.y (mgr.34101) 690 : cluster 0 pgmap v1214: 105 pgs: 105 active+clean; 213 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 842 KiB/s rd, 3.7 MiB/s wr, 51 op/s 2024-03-07T03:36:59.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:36:58 smithi195 bash[66542]: cluster 2024-03-07T03:36:56.949120+0000 mgr.y (mgr.34101) 690 : cluster 0 pgmap v1214: 105 pgs: 105 active+clean; 213 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 842 KiB/s rd, 3.7 MiB/s wr, 51 op/s 2024-03-07T03:37:01.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:00 smithi148 bash[79819]: cluster 2024-03-07T03:36:58.950140+0000 mgr.y (mgr.34101) 691 : cluster 0 pgmap v1216: 105 pgs: 105 active+clean; 229 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 923 KiB/s rd, 3.7 MiB/s wr, 29 op/s 2024-03-07T03:37:01.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:00 smithi148 bash[82222]: cluster 2024-03-07T03:36:58.950140+0000 mgr.y (mgr.34101) 691 : cluster 0 pgmap v1216: 105 pgs: 105 active+clean; 229 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 923 KiB/s rd, 3.7 MiB/s wr, 29 op/s 2024-03-07T03:37:01.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:00 smithi195 bash[66542]: cluster 2024-03-07T03:36:58.950140+0000 mgr.y (mgr.34101) 691 : cluster 0 pgmap v1216: 105 pgs: 105 active+clean; 229 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 923 KiB/s rd, 3.7 MiB/s wr, 29 op/s 2024-03-07T03:37:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:37:02 smithi195 bash[59134]: ts=2024-03-07T03:37:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:02 smithi148 bash[82222]: cluster 2024-03-07T03:37:00.951017+0000 mgr.y (mgr.34101) 692 : cluster 0 pgmap v1217: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-03-07T03:37:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:02 smithi148 bash[82222]: cluster 2024-03-07T03:37:01.737449+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2024-03-07T03:37:03.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:02 smithi148 bash[79819]: cluster 2024-03-07T03:37:00.951017+0000 mgr.y (mgr.34101) 692 : cluster 0 pgmap v1217: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-03-07T03:37:03.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:02 smithi148 bash[79819]: cluster 2024-03-07T03:37:01.737449+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2024-03-07T03:37:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:02 smithi195 bash[66542]: cluster 2024-03-07T03:37:00.951017+0000 mgr.y (mgr.34101) 692 : cluster 0 pgmap v1217: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-03-07T03:37:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:02 smithi195 bash[66542]: cluster 2024-03-07T03:37:01.737449+0000 mon.a (mon.0) 1566 : cluster 0 osdmap e907: 8 total, 8 up, 8 in 2024-03-07T03:37:04.264 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/2.ShrinkToOneAfterSnapshot (11677 ms) 2024-03-07T03:37:04.265 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/2.MinOverlap 2024-03-07T03:37:04.265 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:37:05.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:04 smithi148 bash[79819]: cluster 2024-03-07T03:37:02.951637+0000 mgr.y (mgr.34101) 693 : cluster 0 pgmap v1219: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 3.6 MiB/s wr, 54 op/s 2024-03-07T03:37:05.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:04 smithi148 bash[82222]: cluster 2024-03-07T03:37:02.951637+0000 mgr.y (mgr.34101) 693 : cluster 0 pgmap v1219: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 3.6 MiB/s wr, 54 op/s 2024-03-07T03:37:05.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:04 smithi195 bash[66542]: cluster 2024-03-07T03:37:02.951637+0000 mgr.y (mgr.34101) 693 : cluster 0 pgmap v1219: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 77 KiB/s rd, 3.6 MiB/s wr, 54 op/s 2024-03-07T03:37:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:06 smithi148 bash[79819]: cluster 2024-03-07T03:37:04.952361+0000 mgr.y (mgr.34101) 694 : cluster 0 pgmap v1220: 105 pgs: 105 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2024-03-07T03:37:07.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:06 smithi148 bash[82222]: cluster 2024-03-07T03:37:04.952361+0000 mgr.y (mgr.34101) 694 : cluster 0 pgmap v1220: 105 pgs: 105 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2024-03-07T03:37:07.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:06 smithi195 bash[66542]: cluster 2024-03-07T03:37:04.952361+0000 mgr.y (mgr.34101) 694 : cluster 0 pgmap v1220: 105 pgs: 105 active+clean; 286 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2024-03-07T03:37:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:37:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:37:07] "GET /metrics HTTP/1.1" 200 36728 "" "Prometheus/2.43.0" 2024-03-07T03:37:09.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:08 smithi148 bash[79819]: cluster 2024-03-07T03:37:06.952880+0000 mgr.y (mgr.34101) 695 : cluster 0 pgmap v1221: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 9.1 MiB/s wr, 73 op/s 2024-03-07T03:37:09.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:08 smithi148 bash[82222]: cluster 2024-03-07T03:37:06.952880+0000 mgr.y (mgr.34101) 695 : cluster 0 pgmap v1221: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 9.1 MiB/s wr, 73 op/s 2024-03-07T03:37:09.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:08 smithi195 bash[66542]: cluster 2024-03-07T03:37:06.952880+0000 mgr.y (mgr.34101) 695 : cluster 0 pgmap v1221: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 9.1 MiB/s wr, 73 op/s 2024-03-07T03:37:10.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:09 smithi148 bash[82222]: cluster 2024-03-07T03:37:08.757757+0000 mon.a (mon.0) 1567 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2024-03-07T03:37:10.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:09 smithi148 bash[79819]: cluster 2024-03-07T03:37:08.757757+0000 mon.a (mon.0) 1567 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2024-03-07T03:37:10.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:09 smithi195 bash[66542]: cluster 2024-03-07T03:37:08.757757+0000 mon.a (mon.0) 1567 : cluster 0 osdmap e908: 8 total, 8 up, 8 in 2024-03-07T03:37:11.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:10 smithi148 bash[79819]: cluster 2024-03-07T03:37:08.953895+0000 mgr.y (mgr.34101) 696 : cluster 0 pgmap v1223: 105 pgs: 105 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.1 MiB/s wr, 48 op/s 2024-03-07T03:37:11.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:10 smithi148 bash[82222]: cluster 2024-03-07T03:37:08.953895+0000 mgr.y (mgr.34101) 696 : cluster 0 pgmap v1223: 105 pgs: 105 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.1 MiB/s wr, 48 op/s 2024-03-07T03:37:11.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:10 smithi195 bash[66542]: cluster 2024-03-07T03:37:08.953895+0000 mgr.y (mgr.34101) 696 : cluster 0 pgmap v1223: 105 pgs: 105 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.1 MiB/s wr, 48 op/s 2024-03-07T03:37:12.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:11 smithi195 bash[66542]: cluster 2024-03-07T03:37:10.954858+0000 mgr.y (mgr.34101) 697 : cluster 0 pgmap v1224: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 9.8 MiB/s wr, 51 op/s 2024-03-07T03:37:12.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:11 smithi148 bash[79819]: cluster 2024-03-07T03:37:10.954858+0000 mgr.y (mgr.34101) 697 : cluster 0 pgmap v1224: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 9.8 MiB/s wr, 51 op/s 2024-03-07T03:37:12.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:11 smithi148 bash[82222]: cluster 2024-03-07T03:37:10.954858+0000 mgr.y (mgr.34101) 697 : cluster 0 pgmap v1224: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 9.8 MiB/s wr, 51 op/s 2024-03-07T03:37:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:37:12 smithi195 bash[59134]: ts=2024-03-07T03:37:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:13.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:12 smithi195 bash[66542]: audit 2024-03-07T03:37:12.333687+0000 mon.a (mon.0) 1568 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:13.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:12 smithi148 bash[79819]: audit 2024-03-07T03:37:12.333687+0000 mon.a (mon.0) 1568 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:13.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:12 smithi148 bash[82222]: audit 2024-03-07T03:37:12.333687+0000 mon.a (mon.0) 1568 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:14.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:13 smithi148 bash[79819]: cluster 2024-03-07T03:37:12.955614+0000 mgr.y (mgr.34101) 698 : cluster 0 pgmap v1225: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 56 op/s 2024-03-07T03:37:14.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:13 smithi148 bash[82222]: cluster 2024-03-07T03:37:12.955614+0000 mgr.y (mgr.34101) 698 : cluster 0 pgmap v1225: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 56 op/s 2024-03-07T03:37:14.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:13 smithi195 bash[66542]: cluster 2024-03-07T03:37:12.955614+0000 mgr.y (mgr.34101) 698 : cluster 0 pgmap v1225: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 56 op/s 2024-03-07T03:37:15.587 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/2.MinOverlap (11322 ms) 2024-03-07T03:37:15.587 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/2.ZeroOverlap 2024-03-07T03:37:15.587 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:37:16.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:16 smithi148 bash[79819]: cluster 2024-03-07T03:37:14.956700+0000 mgr.y (mgr.34101) 699 : cluster 0 pgmap v1226: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.9 MiB/s wr, 48 op/s 2024-03-07T03:37:16.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:16 smithi148 bash[82222]: cluster 2024-03-07T03:37:14.956700+0000 mgr.y (mgr.34101) 699 : cluster 0 pgmap v1226: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.9 MiB/s wr, 48 op/s 2024-03-07T03:37:16.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:16 smithi195 bash[66542]: cluster 2024-03-07T03:37:14.956700+0000 mgr.y (mgr.34101) 699 : cluster 0 pgmap v1226: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.9 MiB/s wr, 48 op/s 2024-03-07T03:37:17.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:37:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:37:17] "GET /metrics HTTP/1.1" 200 36735 "" "Prometheus/2.43.0" 2024-03-07T03:37:18.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:18 smithi148 bash[79819]: cluster 2024-03-07T03:37:16.957577+0000 mgr.y (mgr.34101) 700 : cluster 0 pgmap v1227: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 3.3 MiB/s wr, 53 op/s 2024-03-07T03:37:18.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:18 smithi148 bash[82222]: cluster 2024-03-07T03:37:16.957577+0000 mgr.y (mgr.34101) 700 : cluster 0 pgmap v1227: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 3.3 MiB/s wr, 53 op/s 2024-03-07T03:37:18.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:18 smithi195 bash[66542]: cluster 2024-03-07T03:37:16.957577+0000 mgr.y (mgr.34101) 700 : cluster 0 pgmap v1227: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 946 KiB/s rd, 3.3 MiB/s wr, 53 op/s 2024-03-07T03:37:20.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:20 smithi148 bash[79819]: cluster 2024-03-07T03:37:18.958564+0000 mgr.y (mgr.34101) 701 : cluster 0 pgmap v1228: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 53 op/s 2024-03-07T03:37:20.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:20 smithi148 bash[82222]: cluster 2024-03-07T03:37:18.958564+0000 mgr.y (mgr.34101) 701 : cluster 0 pgmap v1228: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 53 op/s 2024-03-07T03:37:20.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:20 smithi195 bash[66542]: cluster 2024-03-07T03:37:18.958564+0000 mgr.y (mgr.34101) 701 : cluster 0 pgmap v1228: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.8 MiB/s wr, 53 op/s 2024-03-07T03:37:21.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:21 smithi148 bash[82222]: cluster 2024-03-07T03:37:20.039857+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2024-03-07T03:37:21.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:21 smithi148 bash[79819]: cluster 2024-03-07T03:37:20.039857+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2024-03-07T03:37:21.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:21 smithi195 bash[66542]: cluster 2024-03-07T03:37:20.039857+0000 mon.a (mon.0) 1569 : cluster 0 osdmap e909: 8 total, 8 up, 8 in 2024-03-07T03:37:22.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:22 smithi148 bash[79819]: cluster 2024-03-07T03:37:20.959345+0000 mgr.y (mgr.34101) 702 : cluster 0 pgmap v1230: 105 pgs: 105 active+clean; 347 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 53 op/s 2024-03-07T03:37:22.291 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:22 smithi148 bash[82222]: cluster 2024-03-07T03:37:20.959345+0000 mgr.y (mgr.34101) 702 : cluster 0 pgmap v1230: 105 pgs: 105 active+clean; 347 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 53 op/s 2024-03-07T03:37:22.305 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:22 smithi195 bash[66542]: cluster 2024-03-07T03:37:20.959345+0000 mgr.y (mgr.34101) 702 : cluster 0 pgmap v1230: 105 pgs: 105 active+clean; 347 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 53 op/s 2024-03-07T03:37:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:37:22 smithi195 bash[59134]: ts=2024-03-07T03:37:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:24.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:24 smithi148 bash[79819]: cluster 2024-03-07T03:37:22.960074+0000 mgr.y (mgr.34101) 703 : cluster 0 pgmap v1231: 105 pgs: 105 active+clean; 351 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 46 op/s 2024-03-07T03:37:24.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:24 smithi148 bash[82222]: cluster 2024-03-07T03:37:22.960074+0000 mgr.y (mgr.34101) 703 : cluster 0 pgmap v1231: 105 pgs: 105 active+clean; 351 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 46 op/s 2024-03-07T03:37:24.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:24 smithi195 bash[66542]: cluster 2024-03-07T03:37:22.960074+0000 mgr.y (mgr.34101) 703 : cluster 0 pgmap v1231: 105 pgs: 105 active+clean; 351 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.4 MiB/s wr, 46 op/s 2024-03-07T03:37:26.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:26 smithi148 bash[79819]: cluster 2024-03-07T03:37:24.961187+0000 mgr.y (mgr.34101) 704 : cluster 0 pgmap v1232: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-03-07T03:37:26.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:26 smithi148 bash[82222]: cluster 2024-03-07T03:37:24.961187+0000 mgr.y (mgr.34101) 704 : cluster 0 pgmap v1232: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-03-07T03:37:26.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:26 smithi195 bash[66542]: cluster 2024-03-07T03:37:24.961187+0000 mgr.y (mgr.34101) 704 : cluster 0 pgmap v1232: 105 pgs: 105 active+clean; 368 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-03-07T03:37:26.722 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/2.ZeroOverlap (11135 ms) 2024-03-07T03:37:27.072 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/2 (82337 ms total) 2024-03-07T03:37:27.072 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:37:27.073 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/3, where TypeParam = LUKS1UnderLUKS1 2024-03-07T03:37:27.129 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/3.Simple 2024-03-07T03:37:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:37:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:37:27] "GET /metrics HTTP/1.1" 200 36735 "" "Prometheus/2.43.0" 2024-03-07T03:37:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:28 smithi195 bash[66542]: cluster 2024-03-07T03:37:26.962114+0000 mgr.y (mgr.34101) 705 : cluster 0 pgmap v1233: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-03-07T03:37:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:28 smithi195 bash[66542]: cluster 2024-03-07T03:37:27.070267+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2024-03-07T03:37:28.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:28 smithi195 bash[66542]: audit 2024-03-07T03:37:27.580571+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:28.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:28 smithi148 bash[79819]: cluster 2024-03-07T03:37:26.962114+0000 mgr.y (mgr.34101) 705 : cluster 0 pgmap v1233: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-03-07T03:37:28.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:28 smithi148 bash[79819]: cluster 2024-03-07T03:37:27.070267+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2024-03-07T03:37:28.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:28 smithi148 bash[79819]: audit 2024-03-07T03:37:27.580571+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:28.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:28 smithi148 bash[82222]: cluster 2024-03-07T03:37:26.962114+0000 mgr.y (mgr.34101) 705 : cluster 0 pgmap v1233: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-03-07T03:37:28.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:28 smithi148 bash[82222]: cluster 2024-03-07T03:37:27.070267+0000 mon.a (mon.0) 1570 : cluster 0 osdmap e910: 8 total, 8 up, 8 in 2024-03-07T03:37:28.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:28 smithi148 bash[82222]: audit 2024-03-07T03:37:27.580571+0000 mon.a (mon.0) 1571 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:29.086 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:37:29.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:29 smithi148 bash[79819]: cluster 2024-03-07T03:37:28.082441+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2024-03-07T03:37:29.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:29 smithi148 bash[79819]: audit 2024-03-07T03:37:28.087985+0000 mon.a (mon.0) 1573 : audit 1 from='client.? 172.21.15.148:0/1285500220' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:37:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:29 smithi148 bash[82222]: cluster 2024-03-07T03:37:28.082441+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2024-03-07T03:37:29.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:29 smithi148 bash[82222]: audit 2024-03-07T03:37:28.087985+0000 mon.a (mon.0) 1573 : audit 1 from='client.? 172.21.15.148:0/1285500220' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:37:29.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:29 smithi195 bash[66542]: cluster 2024-03-07T03:37:28.082441+0000 mon.a (mon.0) 1572 : cluster 0 osdmap e911: 8 total, 8 up, 8 in 2024-03-07T03:37:29.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:29 smithi195 bash[66542]: audit 2024-03-07T03:37:28.087985+0000 mon.a (mon.0) 1573 : audit 1 from='client.? 172.21.15.148:0/1285500220' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-39","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:37:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[79819]: cluster 2024-03-07T03:37:28.963280+0000 mgr.y (mgr.34101) 706 : cluster 0 pgmap v1236: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[79819]: cluster 2024-03-07T03:37:29.075429+0000 mon.a (mon.0) 1574 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:37:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[79819]: audit 2024-03-07T03:37:29.085611+0000 mon.a (mon.0) 1575 : audit 1 from='client.? 172.21.15.148:0/1285500220' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:37:30.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[79819]: cluster 2024-03-07T03:37:29.092934+0000 mon.a (mon.0) 1576 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2024-03-07T03:37:30.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[82222]: cluster 2024-03-07T03:37:28.963280+0000 mgr.y (mgr.34101) 706 : cluster 0 pgmap v1236: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:30.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[82222]: cluster 2024-03-07T03:37:29.075429+0000 mon.a (mon.0) 1574 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:37:30.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[82222]: audit 2024-03-07T03:37:29.085611+0000 mon.a (mon.0) 1575 : audit 1 from='client.? 172.21.15.148:0/1285500220' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:37:30.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:30 smithi148 bash[82222]: cluster 2024-03-07T03:37:29.092934+0000 mon.a (mon.0) 1576 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2024-03-07T03:37:30.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:30 smithi195 bash[66542]: cluster 2024-03-07T03:37:28.963280+0000 mgr.y (mgr.34101) 706 : cluster 0 pgmap v1236: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:30.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:30 smithi195 bash[66542]: cluster 2024-03-07T03:37:29.075429+0000 mon.a (mon.0) 1574 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:37:30.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:30 smithi195 bash[66542]: audit 2024-03-07T03:37:29.085611+0000 mon.a (mon.0) 1575 : audit 1 from='client.? 172.21.15.148:0/1285500220' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:37:30.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:30 smithi195 bash[66542]: cluster 2024-03-07T03:37:29.092934+0000 mon.a (mon.0) 1576 : cluster 0 osdmap e912: 8 total, 8 up, 8 in 2024-03-07T03:37:31.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:31 smithi148 bash[79819]: cluster 2024-03-07T03:37:30.102965+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2024-03-07T03:37:31.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:31 smithi148 bash[82222]: cluster 2024-03-07T03:37:30.102965+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2024-03-07T03:37:31.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:31 smithi195 bash[66542]: cluster 2024-03-07T03:37:30.102965+0000 mon.a (mon.0) 1577 : cluster 0 osdmap e913: 8 total, 8 up, 8 in 2024-03-07T03:37:32.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:32 smithi148 bash[79819]: cluster 2024-03-07T03:37:30.964053+0000 mgr.y (mgr.34101) 707 : cluster 0 pgmap v1239: 105 pgs: 19 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 998 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:32.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:32 smithi148 bash[79819]: cluster 2024-03-07T03:37:31.096166+0000 mon.a (mon.0) 1578 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:37:32.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:32 smithi148 bash[79819]: cluster 2024-03-07T03:37:31.113735+0000 mon.a (mon.0) 1579 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2024-03-07T03:37:32.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:32 smithi148 bash[82222]: cluster 2024-03-07T03:37:30.964053+0000 mgr.y (mgr.34101) 707 : cluster 0 pgmap v1239: 105 pgs: 19 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 998 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:32.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:32 smithi148 bash[82222]: cluster 2024-03-07T03:37:31.096166+0000 mon.a (mon.0) 1578 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:37:32.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:32 smithi148 bash[82222]: cluster 2024-03-07T03:37:31.113735+0000 mon.a (mon.0) 1579 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2024-03-07T03:37:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:37:32 smithi195 bash[59134]: ts=2024-03-07T03:37:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:32.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:32 smithi195 bash[66542]: cluster 2024-03-07T03:37:30.964053+0000 mgr.y (mgr.34101) 707 : cluster 0 pgmap v1239: 105 pgs: 19 creating+peering, 4 unknown, 82 active+clean; 582 KiB data, 998 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:32.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:32 smithi195 bash[66542]: cluster 2024-03-07T03:37:31.096166+0000 mon.a (mon.0) 1578 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:37:32.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:32 smithi195 bash[66542]: cluster 2024-03-07T03:37:31.113735+0000 mon.a (mon.0) 1579 : cluster 0 osdmap e914: 8 total, 8 up, 8 in 2024-03-07T03:37:34.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:34 smithi148 bash[82222]: cluster 2024-03-07T03:37:32.964750+0000 mgr.y (mgr.34101) 708 : cluster 0 pgmap v1241: 105 pgs: 19 creating+peering, 86 active+clean; 582 KiB data, 840 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:34.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:34 smithi148 bash[79819]: cluster 2024-03-07T03:37:32.964750+0000 mgr.y (mgr.34101) 708 : cluster 0 pgmap v1241: 105 pgs: 19 creating+peering, 86 active+clean; 582 KiB data, 840 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:34.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:34 smithi195 bash[66542]: cluster 2024-03-07T03:37:32.964750+0000 mgr.y (mgr.34101) 708 : cluster 0 pgmap v1241: 105 pgs: 19 creating+peering, 86 active+clean; 582 KiB data, 840 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:37:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:36 smithi148 bash[79819]: cluster 2024-03-07T03:37:34.966033+0000 mgr.y (mgr.34101) 709 : cluster 0 pgmap v1242: 105 pgs: 105 active+clean; 582 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:37:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:36 smithi148 bash[79819]: cluster 2024-03-07T03:37:35.126022+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2024-03-07T03:37:36.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:36 smithi148 bash[82222]: cluster 2024-03-07T03:37:34.966033+0000 mgr.y (mgr.34101) 709 : cluster 0 pgmap v1242: 105 pgs: 105 active+clean; 582 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:37:36.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:36 smithi148 bash[82222]: cluster 2024-03-07T03:37:35.126022+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2024-03-07T03:37:36.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:36 smithi195 bash[66542]: cluster 2024-03-07T03:37:34.966033+0000 mgr.y (mgr.34101) 709 : cluster 0 pgmap v1242: 105 pgs: 105 active+clean; 582 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-03-07T03:37:36.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:36 smithi195 bash[66542]: cluster 2024-03-07T03:37:35.126022+0000 mon.a (mon.0) 1580 : cluster 0 osdmap e915: 8 total, 8 up, 8 in 2024-03-07T03:37:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:37:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:37:37] "GET /metrics HTTP/1.1" 200 34301 "" "Prometheus/2.43.0" 2024-03-07T03:37:39.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:38 smithi148 bash[82222]: cluster 2024-03-07T03:37:36.966992+0000 mgr.y (mgr.34101) 710 : cluster 0 pgmap v1244: 105 pgs: 105 active+clean; 1.6 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 150 KiB/s wr, 7 op/s 2024-03-07T03:37:39.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:38 smithi148 bash[82222]: cluster 2024-03-07T03:37:37.586767+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2024-03-07T03:37:39.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:38 smithi148 bash[79819]: cluster 2024-03-07T03:37:36.966992+0000 mgr.y (mgr.34101) 710 : cluster 0 pgmap v1244: 105 pgs: 105 active+clean; 1.6 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 150 KiB/s wr, 7 op/s 2024-03-07T03:37:39.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:38 smithi148 bash[79819]: cluster 2024-03-07T03:37:37.586767+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2024-03-07T03:37:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:38 smithi195 bash[66542]: cluster 2024-03-07T03:37:36.966992+0000 mgr.y (mgr.34101) 710 : cluster 0 pgmap v1244: 105 pgs: 105 active+clean; 1.6 MiB data, 563 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 150 KiB/s wr, 7 op/s 2024-03-07T03:37:39.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:38 smithi195 bash[66542]: cluster 2024-03-07T03:37:37.586767+0000 mon.a (mon.0) 1581 : cluster 0 osdmap e916: 8 total, 8 up, 8 in 2024-03-07T03:37:40.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:39 smithi148 bash[82222]: audit 2024-03-07T03:37:38.622141+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:37:40.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:39 smithi148 bash[79819]: audit 2024-03-07T03:37:38.622141+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:37:40.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:39 smithi195 bash[66542]: audit 2024-03-07T03:37:38.622141+0000 mon.a (mon.0) 1582 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:37:41.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:40 smithi148 bash[79819]: cluster 2024-03-07T03:37:38.968015+0000 mgr.y (mgr.34101) 711 : cluster 0 pgmap v1246: 105 pgs: 105 active+clean; 18 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 2.2 MiB/s wr, 40 op/s 2024-03-07T03:37:41.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:40 smithi148 bash[82222]: cluster 2024-03-07T03:37:38.968015+0000 mgr.y (mgr.34101) 711 : cluster 0 pgmap v1246: 105 pgs: 105 active+clean; 18 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 2.2 MiB/s wr, 40 op/s 2024-03-07T03:37:41.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:40 smithi195 bash[66542]: cluster 2024-03-07T03:37:38.968015+0000 mgr.y (mgr.34101) 711 : cluster 0 pgmap v1246: 105 pgs: 105 active+clean; 18 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 2.2 MiB/s wr, 40 op/s 2024-03-07T03:37:42.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:37:42 smithi195 bash[59134]: ts=2024-03-07T03:37:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:42 smithi148 bash[79819]: cluster 2024-03-07T03:37:40.968876+0000 mgr.y (mgr.34101) 712 : cluster 0 pgmap v1247: 105 pgs: 105 active+clean; 30 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 3.6 MiB/s wr, 51 op/s 2024-03-07T03:37:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:42 smithi148 bash[79819]: audit 2024-03-07T03:37:42.334497+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:42 smithi148 bash[82222]: cluster 2024-03-07T03:37:40.968876+0000 mgr.y (mgr.34101) 712 : cluster 0 pgmap v1247: 105 pgs: 105 active+clean; 30 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 3.6 MiB/s wr, 51 op/s 2024-03-07T03:37:43.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:42 smithi148 bash[82222]: audit 2024-03-07T03:37:42.334497+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:42 smithi195 bash[66542]: cluster 2024-03-07T03:37:40.968876+0000 mgr.y (mgr.34101) 712 : cluster 0 pgmap v1247: 105 pgs: 105 active+clean; 30 MiB data, 625 MiB used, 715 GiB / 715 GiB avail; 37 KiB/s rd, 3.6 MiB/s wr, 51 op/s 2024-03-07T03:37:43.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:42 smithi195 bash[66542]: audit 2024-03-07T03:37:42.334497+0000 mon.a (mon.0) 1583 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:45.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:44 smithi148 bash[79819]: cluster 2024-03-07T03:37:42.969617+0000 mgr.y (mgr.34101) 713 : cluster 0 pgmap v1248: 105 pgs: 105 active+clean; 30 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-03-07T03:37:45.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:44 smithi148 bash[82222]: cluster 2024-03-07T03:37:42.969617+0000 mgr.y (mgr.34101) 713 : cluster 0 pgmap v1248: 105 pgs: 105 active+clean; 30 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-03-07T03:37:45.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:44 smithi195 bash[66542]: cluster 2024-03-07T03:37:42.969617+0000 mgr.y (mgr.34101) 713 : cluster 0 pgmap v1248: 105 pgs: 105 active+clean; 30 MiB data, 626 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-03-07T03:37:45.784 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/3.Simple (18656 ms) 2024-03-07T03:37:45.784 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/3.Grow 2024-03-07T03:37:45.784 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:37:47.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:46 smithi148 bash[79819]: cluster 2024-03-07T03:37:44.970777+0000 mgr.y (mgr.34101) 714 : cluster 0 pgmap v1249: 105 pgs: 105 active+clean; 55 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.9 MiB/s wr, 45 op/s 2024-03-07T03:37:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:46 smithi148 bash[82222]: cluster 2024-03-07T03:37:44.970777+0000 mgr.y (mgr.34101) 714 : cluster 0 pgmap v1249: 105 pgs: 105 active+clean; 55 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.9 MiB/s wr, 45 op/s 2024-03-07T03:37:47.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:46 smithi195 bash[66542]: cluster 2024-03-07T03:37:44.970777+0000 mgr.y (mgr.34101) 714 : cluster 0 pgmap v1249: 105 pgs: 105 active+clean; 55 MiB data, 668 MiB used, 715 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.9 MiB/s wr, 45 op/s 2024-03-07T03:37:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:37:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:37:47] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.43.0" 2024-03-07T03:37:49.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:48 smithi148 bash[79819]: cluster 2024-03-07T03:37:46.971669+0000 mgr.y (mgr.34101) 715 : cluster 0 pgmap v1250: 105 pgs: 105 active+clean; 59 MiB data, 695 MiB used, 715 GiB / 715 GiB avail; 7.6 MiB/s rd, 6.1 MiB/s wr, 51 op/s 2024-03-07T03:37:49.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:48 smithi148 bash[82222]: cluster 2024-03-07T03:37:46.971669+0000 mgr.y (mgr.34101) 715 : cluster 0 pgmap v1250: 105 pgs: 105 active+clean; 59 MiB data, 695 MiB used, 715 GiB / 715 GiB avail; 7.6 MiB/s rd, 6.1 MiB/s wr, 51 op/s 2024-03-07T03:37:49.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:48 smithi195 bash[66542]: cluster 2024-03-07T03:37:46.971669+0000 mgr.y (mgr.34101) 715 : cluster 0 pgmap v1250: 105 pgs: 105 active+clean; 59 MiB data, 695 MiB used, 715 GiB / 715 GiB avail; 7.6 MiB/s rd, 6.1 MiB/s wr, 51 op/s 2024-03-07T03:37:51.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:50 smithi148 bash[79819]: cluster 2024-03-07T03:37:48.972698+0000 mgr.y (mgr.34101) 716 : cluster 0 pgmap v1251: 105 pgs: 105 active+clean; 59 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.4 MiB/s wr, 55 op/s 2024-03-07T03:37:51.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:50 smithi148 bash[82222]: cluster 2024-03-07T03:37:48.972698+0000 mgr.y (mgr.34101) 716 : cluster 0 pgmap v1251: 105 pgs: 105 active+clean; 59 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.4 MiB/s wr, 55 op/s 2024-03-07T03:37:51.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:50 smithi195 bash[66542]: cluster 2024-03-07T03:37:48.972698+0000 mgr.y (mgr.34101) 716 : cluster 0 pgmap v1251: 105 pgs: 105 active+clean; 59 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 9.2 MiB/s rd, 5.4 MiB/s wr, 55 op/s 2024-03-07T03:37:52.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:51 smithi148 bash[79819]: cluster 2024-03-07T03:37:50.661759+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2024-03-07T03:37:52.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:51 smithi148 bash[82222]: cluster 2024-03-07T03:37:50.661759+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2024-03-07T03:37:52.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:51 smithi195 bash[66542]: cluster 2024-03-07T03:37:50.661759+0000 mon.a (mon.0) 1584 : cluster 0 osdmap e917: 8 total, 8 up, 8 in 2024-03-07T03:37:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:37:52 smithi195 bash[59134]: ts=2024-03-07T03:37:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:37:53.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:52 smithi148 bash[82222]: cluster 2024-03-07T03:37:50.973296+0000 mgr.y (mgr.34101) 717 : cluster 0 pgmap v1253: 105 pgs: 105 active+clean; 59 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2024-03-07T03:37:53.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:52 smithi148 bash[79819]: cluster 2024-03-07T03:37:50.973296+0000 mgr.y (mgr.34101) 717 : cluster 0 pgmap v1253: 105 pgs: 105 active+clean; 59 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2024-03-07T03:37:53.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:52 smithi195 bash[66542]: cluster 2024-03-07T03:37:50.973296+0000 mgr.y (mgr.34101) 717 : cluster 0 pgmap v1253: 105 pgs: 105 active+clean; 59 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 28 op/s 2024-03-07T03:37:55.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:54 smithi148 bash[79819]: cluster 2024-03-07T03:37:52.974140+0000 mgr.y (mgr.34101) 718 : cluster 0 pgmap v1254: 105 pgs: 105 active+clean; 63 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2024-03-07T03:37:55.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:54 smithi148 bash[82222]: cluster 2024-03-07T03:37:52.974140+0000 mgr.y (mgr.34101) 718 : cluster 0 pgmap v1254: 105 pgs: 105 active+clean; 63 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2024-03-07T03:37:55.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:54 smithi195 bash[66542]: cluster 2024-03-07T03:37:52.974140+0000 mgr.y (mgr.34101) 718 : cluster 0 pgmap v1254: 105 pgs: 105 active+clean; 63 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 28 op/s 2024-03-07T03:37:57.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:56 smithi148 bash[82222]: cluster 2024-03-07T03:37:54.975306+0000 mgr.y (mgr.34101) 719 : cluster 0 pgmap v1255: 105 pgs: 105 active+clean; 88 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.3 MiB/s wr, 45 op/s 2024-03-07T03:37:57.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:56 smithi148 bash[79819]: cluster 2024-03-07T03:37:54.975306+0000 mgr.y (mgr.34101) 719 : cluster 0 pgmap v1255: 105 pgs: 105 active+clean; 88 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.3 MiB/s wr, 45 op/s 2024-03-07T03:37:57.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:56 smithi195 bash[66542]: cluster 2024-03-07T03:37:54.975306+0000 mgr.y (mgr.34101) 719 : cluster 0 pgmap v1255: 105 pgs: 105 active+clean; 88 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 4.0 MiB/s rd, 3.3 MiB/s wr, 45 op/s 2024-03-07T03:37:57.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:37:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:37:57] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.43.0" 2024-03-07T03:37:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:57 smithi148 bash[82222]: audit 2024-03-07T03:37:56.892210+0000 mon.a (mon.0) 1585 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:37:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:57 smithi148 bash[82222]: audit 2024-03-07T03:37:57.333914+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:57 smithi148 bash[79819]: audit 2024-03-07T03:37:56.892210+0000 mon.a (mon.0) 1585 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:37:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:57 smithi148 bash[79819]: audit 2024-03-07T03:37:57.333914+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:57 smithi195 bash[66542]: audit 2024-03-07T03:37:56.892210+0000 mon.a (mon.0) 1585 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:37:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:57 smithi195 bash[66542]: audit 2024-03-07T03:37:57.333914+0000 mon.a (mon.0) 1586 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:37:59.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:37:58 smithi148 bash[79819]: cluster 2024-03-07T03:37:56.976215+0000 mgr.y (mgr.34101) 720 : cluster 0 pgmap v1256: 105 pgs: 105 active+clean; 88 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.9 MiB/s wr, 48 op/s 2024-03-07T03:37:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:37:58 smithi148 bash[82222]: cluster 2024-03-07T03:37:56.976215+0000 mgr.y (mgr.34101) 720 : cluster 0 pgmap v1256: 105 pgs: 105 active+clean; 88 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.9 MiB/s wr, 48 op/s 2024-03-07T03:37:59.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:37:58 smithi195 bash[66542]: cluster 2024-03-07T03:37:56.976215+0000 mgr.y (mgr.34101) 720 : cluster 0 pgmap v1256: 105 pgs: 105 active+clean; 88 MiB data, 800 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 2.9 MiB/s wr, 48 op/s 2024-03-07T03:38:01.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:00 smithi148 bash[79819]: cluster 2024-03-07T03:37:58.977166+0000 mgr.y (mgr.34101) 721 : cluster 0 pgmap v1257: 105 pgs: 105 active+clean; 92 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 191 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:38:01.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:00 smithi148 bash[82222]: cluster 2024-03-07T03:37:58.977166+0000 mgr.y (mgr.34101) 721 : cluster 0 pgmap v1257: 105 pgs: 105 active+clean; 92 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 191 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:38:01.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:00 smithi195 bash[66542]: cluster 2024-03-07T03:37:58.977166+0000 mgr.y (mgr.34101) 721 : cluster 0 pgmap v1257: 105 pgs: 105 active+clean; 92 MiB data, 804 MiB used, 714 GiB / 715 GiB avail; 191 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:38:01.362 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/3.Grow (15578 ms) 2024-03-07T03:38:01.363 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/3.Shrink 2024-03-07T03:38:01.363 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:38:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:38:02 smithi195 bash[59134]: ts=2024-03-07T03:38:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:02 smithi148 bash[82222]: cluster 2024-03-07T03:38:00.978038+0000 mgr.y (mgr.34101) 722 : cluster 0 pgmap v1258: 105 pgs: 105 active+clean; 96 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 47 op/s 2024-03-07T03:38:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:02 smithi148 bash[82222]: audit 2024-03-07T03:38:02.685051+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:03.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:02 smithi148 bash[79819]: cluster 2024-03-07T03:38:00.978038+0000 mgr.y (mgr.34101) 722 : cluster 0 pgmap v1258: 105 pgs: 105 active+clean; 96 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 47 op/s 2024-03-07T03:38:03.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:02 smithi148 bash[79819]: audit 2024-03-07T03:38:02.685051+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:02 smithi195 bash[66542]: cluster 2024-03-07T03:38:00.978038+0000 mgr.y (mgr.34101) 722 : cluster 0 pgmap v1258: 105 pgs: 105 active+clean; 96 MiB data, 825 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.6 MiB/s wr, 47 op/s 2024-03-07T03:38:03.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:02 smithi195 bash[66542]: audit 2024-03-07T03:38:02.685051+0000 mon.a (mon.0) 1587 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[82222]: audit 2024-03-07T03:38:02.694470+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[82222]: audit 2024-03-07T03:38:02.936736+0000 mon.a (mon.0) 1589 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[82222]: audit 2024-03-07T03:38:02.946455+0000 mon.a (mon.0) 1590 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[82222]: audit 2024-03-07T03:38:03.525978+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:38:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[82222]: audit 2024-03-07T03:38:03.527602+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:38:04.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[82222]: audit 2024-03-07T03:38:03.537193+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[79819]: audit 2024-03-07T03:38:02.694470+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[79819]: audit 2024-03-07T03:38:02.936736+0000 mon.a (mon.0) 1589 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[79819]: audit 2024-03-07T03:38:02.946455+0000 mon.a (mon.0) 1590 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[79819]: audit 2024-03-07T03:38:03.525978+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:38:04.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[79819]: audit 2024-03-07T03:38:03.527602+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:38:04.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:03 smithi148 bash[79819]: audit 2024-03-07T03:38:03.537193+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:03 smithi195 bash[66542]: audit 2024-03-07T03:38:02.694470+0000 mon.a (mon.0) 1588 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:03 smithi195 bash[66542]: audit 2024-03-07T03:38:02.936736+0000 mon.a (mon.0) 1589 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:03 smithi195 bash[66542]: audit 2024-03-07T03:38:02.946455+0000 mon.a (mon.0) 1590 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:04.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:03 smithi195 bash[66542]: audit 2024-03-07T03:38:03.525978+0000 mon.a (mon.0) 1591 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:38:04.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:03 smithi195 bash[66542]: audit 2024-03-07T03:38:03.527602+0000 mon.a (mon.0) 1592 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:38:04.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:03 smithi195 bash[66542]: audit 2024-03-07T03:38:03.537193+0000 mon.a (mon.0) 1593 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:38:05.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:04 smithi148 bash[82222]: cluster 2024-03-07T03:38:02.978559+0000 mgr.y (mgr.34101) 723 : cluster 0 pgmap v1259: 105 pgs: 105 active+clean; 104 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.8 MiB/s wr, 41 op/s 2024-03-07T03:38:05.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:04 smithi148 bash[79819]: cluster 2024-03-07T03:38:02.978559+0000 mgr.y (mgr.34101) 723 : cluster 0 pgmap v1259: 105 pgs: 105 active+clean; 104 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.8 MiB/s wr, 41 op/s 2024-03-07T03:38:05.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:04 smithi195 bash[66542]: cluster 2024-03-07T03:38:02.978559+0000 mgr.y (mgr.34101) 723 : cluster 0 pgmap v1259: 105 pgs: 105 active+clean; 104 MiB data, 834 MiB used, 714 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.8 MiB/s wr, 41 op/s 2024-03-07T03:38:07.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:06 smithi148 bash[82222]: cluster 2024-03-07T03:38:04.979664+0000 mgr.y (mgr.34101) 724 : cluster 0 pgmap v1260: 105 pgs: 105 active+clean; 117 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.8 MiB/s wr, 54 op/s 2024-03-07T03:38:07.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:06 smithi148 bash[82222]: cluster 2024-03-07T03:38:05.713875+0000 mon.a (mon.0) 1594 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2024-03-07T03:38:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:06 smithi148 bash[79819]: cluster 2024-03-07T03:38:04.979664+0000 mgr.y (mgr.34101) 724 : cluster 0 pgmap v1260: 105 pgs: 105 active+clean; 117 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.8 MiB/s wr, 54 op/s 2024-03-07T03:38:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:06 smithi148 bash[79819]: cluster 2024-03-07T03:38:05.713875+0000 mon.a (mon.0) 1594 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2024-03-07T03:38:07.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:06 smithi195 bash[66542]: cluster 2024-03-07T03:38:04.979664+0000 mgr.y (mgr.34101) 724 : cluster 0 pgmap v1260: 105 pgs: 105 active+clean; 117 MiB data, 883 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.8 MiB/s wr, 54 op/s 2024-03-07T03:38:07.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:06 smithi195 bash[66542]: cluster 2024-03-07T03:38:05.713875+0000 mon.a (mon.0) 1594 : cluster 0 osdmap e918: 8 total, 8 up, 8 in 2024-03-07T03:38:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:38:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:38:07] "GET /metrics HTTP/1.1" 200 36720 "" "Prometheus/2.43.0" 2024-03-07T03:38:09.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:08 smithi148 bash[79819]: cluster 2024-03-07T03:38:06.980469+0000 mgr.y (mgr.34101) 725 : cluster 0 pgmap v1262: 105 pgs: 105 active+clean; 121 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-03-07T03:38:09.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:08 smithi148 bash[82222]: cluster 2024-03-07T03:38:06.980469+0000 mgr.y (mgr.34101) 725 : cluster 0 pgmap v1262: 105 pgs: 105 active+clean; 121 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-03-07T03:38:09.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:08 smithi195 bash[66542]: cluster 2024-03-07T03:38:06.980469+0000 mgr.y (mgr.34101) 725 : cluster 0 pgmap v1262: 105 pgs: 105 active+clean; 121 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 30 op/s 2024-03-07T03:38:11.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:10 smithi148 bash[79819]: cluster 2024-03-07T03:38:08.981428+0000 mgr.y (mgr.34101) 726 : cluster 0 pgmap v1263: 105 pgs: 105 active+clean; 138 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 57 op/s 2024-03-07T03:38:11.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:10 smithi148 bash[82222]: cluster 2024-03-07T03:38:08.981428+0000 mgr.y (mgr.34101) 726 : cluster 0 pgmap v1263: 105 pgs: 105 active+clean; 138 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 57 op/s 2024-03-07T03:38:11.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:10 smithi195 bash[66542]: cluster 2024-03-07T03:38:08.981428+0000 mgr.y (mgr.34101) 726 : cluster 0 pgmap v1263: 105 pgs: 105 active+clean; 138 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.0 MiB/s wr, 57 op/s 2024-03-07T03:38:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:38:12 smithi195 bash[59134]: ts=2024-03-07T03:38:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:12 smithi148 bash[82222]: cluster 2024-03-07T03:38:10.982303+0000 mgr.y (mgr.34101) 727 : cluster 0 pgmap v1264: 105 pgs: 105 active+clean; 146 MiB data, 970 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-03-07T03:38:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:12 smithi148 bash[82222]: audit 2024-03-07T03:38:12.335112+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:12 smithi148 bash[79819]: cluster 2024-03-07T03:38:10.982303+0000 mgr.y (mgr.34101) 727 : cluster 0 pgmap v1264: 105 pgs: 105 active+clean; 146 MiB data, 970 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-03-07T03:38:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:12 smithi148 bash[79819]: audit 2024-03-07T03:38:12.335112+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:13.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:12 smithi195 bash[66542]: cluster 2024-03-07T03:38:10.982303+0000 mgr.y (mgr.34101) 727 : cluster 0 pgmap v1264: 105 pgs: 105 active+clean; 146 MiB data, 970 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-03-07T03:38:13.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:12 smithi195 bash[66542]: audit 2024-03-07T03:38:12.335112+0000 mon.a (mon.0) 1595 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:15.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:14 smithi148 bash[79819]: cluster 2024-03-07T03:38:12.983029+0000 mgr.y (mgr.34101) 728 : cluster 0 pgmap v1265: 105 pgs: 105 active+clean; 146 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 4.6 MiB/s wr, 49 op/s 2024-03-07T03:38:15.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:14 smithi148 bash[82222]: cluster 2024-03-07T03:38:12.983029+0000 mgr.y (mgr.34101) 728 : cluster 0 pgmap v1265: 105 pgs: 105 active+clean; 146 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 4.6 MiB/s wr, 49 op/s 2024-03-07T03:38:15.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:14 smithi195 bash[66542]: cluster 2024-03-07T03:38:12.983029+0000 mgr.y (mgr.34101) 728 : cluster 0 pgmap v1265: 105 pgs: 105 active+clean; 146 MiB data, 974 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 4.6 MiB/s wr, 49 op/s 2024-03-07T03:38:16.226 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/3.Shrink (14863 ms) 2024-03-07T03:38:16.226 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/3.ShrinkToOne 2024-03-07T03:38:16.226 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:38:17.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:16 smithi148 bash[79819]: cluster 2024-03-07T03:38:14.983692+0000 mgr.y (mgr.34101) 729 : cluster 0 pgmap v1266: 105 pgs: 105 active+clean; 166 MiB data, 995 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 5.3 MiB/s wr, 44 op/s 2024-03-07T03:38:17.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:16 smithi148 bash[82222]: cluster 2024-03-07T03:38:14.983692+0000 mgr.y (mgr.34101) 729 : cluster 0 pgmap v1266: 105 pgs: 105 active+clean; 166 MiB data, 995 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 5.3 MiB/s wr, 44 op/s 2024-03-07T03:38:17.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:16 smithi195 bash[66542]: cluster 2024-03-07T03:38:14.983692+0000 mgr.y (mgr.34101) 729 : cluster 0 pgmap v1266: 105 pgs: 105 active+clean; 166 MiB data, 995 MiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 5.3 MiB/s wr, 44 op/s 2024-03-07T03:38:17.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:38:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:38:17] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:38:19.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:18 smithi148 bash[79819]: cluster 2024-03-07T03:38:16.984208+0000 mgr.y (mgr.34101) 730 : cluster 0 pgmap v1267: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.1 MiB/s wr, 46 op/s 2024-03-07T03:38:19.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:18 smithi148 bash[82222]: cluster 2024-03-07T03:38:16.984208+0000 mgr.y (mgr.34101) 730 : cluster 0 pgmap v1267: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.1 MiB/s wr, 46 op/s 2024-03-07T03:38:19.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:18 smithi195 bash[66542]: cluster 2024-03-07T03:38:16.984208+0000 mgr.y (mgr.34101) 730 : cluster 0 pgmap v1267: 105 pgs: 105 active+clean; 170 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 5.1 MiB/s wr, 46 op/s 2024-03-07T03:38:21.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:20 smithi148 bash[79819]: cluster 2024-03-07T03:38:18.984785+0000 mgr.y (mgr.34101) 731 : cluster 0 pgmap v1268: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 4.8 MiB/s wr, 52 op/s 2024-03-07T03:38:21.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:20 smithi148 bash[82222]: cluster 2024-03-07T03:38:18.984785+0000 mgr.y (mgr.34101) 731 : cluster 0 pgmap v1268: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 4.8 MiB/s wr, 52 op/s 2024-03-07T03:38:21.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:20 smithi195 bash[66542]: cluster 2024-03-07T03:38:18.984785+0000 mgr.y (mgr.34101) 731 : cluster 0 pgmap v1268: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.1 MiB/s rd, 4.8 MiB/s wr, 52 op/s 2024-03-07T03:38:22.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:21 smithi148 bash[79819]: cluster 2024-03-07T03:38:20.753098+0000 mon.a (mon.0) 1596 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2024-03-07T03:38:22.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:21 smithi148 bash[82222]: cluster 2024-03-07T03:38:20.753098+0000 mon.a (mon.0) 1596 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2024-03-07T03:38:22.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:21 smithi195 bash[66542]: cluster 2024-03-07T03:38:20.753098+0000 mon.a (mon.0) 1596 : cluster 0 osdmap e919: 8 total, 8 up, 8 in 2024-03-07T03:38:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:38:22 smithi195 bash[59134]: ts=2024-03-07T03:38:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:23.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:22 smithi148 bash[79819]: cluster 2024-03-07T03:38:20.985263+0000 mgr.y (mgr.34101) 732 : cluster 0 pgmap v1270: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-03-07T03:38:23.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:22 smithi148 bash[82222]: cluster 2024-03-07T03:38:20.985263+0000 mgr.y (mgr.34101) 732 : cluster 0 pgmap v1270: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-03-07T03:38:23.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:22 smithi195 bash[66542]: cluster 2024-03-07T03:38:20.985263+0000 mgr.y (mgr.34101) 732 : cluster 0 pgmap v1270: 105 pgs: 105 active+clean; 175 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-03-07T03:38:25.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:24 smithi148 bash[79819]: cluster 2024-03-07T03:38:22.985722+0000 mgr.y (mgr.34101) 733 : cluster 0 pgmap v1271: 105 pgs: 105 active+clean; 179 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 41 op/s 2024-03-07T03:38:25.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:24 smithi148 bash[82222]: cluster 2024-03-07T03:38:22.985722+0000 mgr.y (mgr.34101) 733 : cluster 0 pgmap v1271: 105 pgs: 105 active+clean; 179 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 41 op/s 2024-03-07T03:38:25.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:24 smithi195 bash[66542]: cluster 2024-03-07T03:38:22.985722+0000 mgr.y (mgr.34101) 733 : cluster 0 pgmap v1271: 105 pgs: 105 active+clean; 179 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 41 op/s 2024-03-07T03:38:27.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:26 smithi148 bash[79819]: cluster 2024-03-07T03:38:24.986861+0000 mgr.y (mgr.34101) 734 : cluster 0 pgmap v1272: 105 pgs: 105 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 3.4 MiB/s wr, 41 op/s 2024-03-07T03:38:27.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:26 smithi148 bash[82222]: cluster 2024-03-07T03:38:24.986861+0000 mgr.y (mgr.34101) 734 : cluster 0 pgmap v1272: 105 pgs: 105 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 3.4 MiB/s wr, 41 op/s 2024-03-07T03:38:27.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:26 smithi195 bash[66542]: cluster 2024-03-07T03:38:24.986861+0000 mgr.y (mgr.34101) 734 : cluster 0 pgmap v1272: 105 pgs: 105 active+clean; 200 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 3.4 MiB/s wr, 41 op/s 2024-03-07T03:38:27.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:38:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:38:27] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:38:28.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:27 smithi148 bash[79819]: audit 2024-03-07T03:38:27.335519+0000 mon.a (mon.0) 1597 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:28.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:27 smithi148 bash[82222]: audit 2024-03-07T03:38:27.335519+0000 mon.a (mon.0) 1597 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:28.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:27 smithi195 bash[66542]: audit 2024-03-07T03:38:27.335519+0000 mon.a (mon.0) 1597 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:29.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:28 smithi148 bash[79819]: cluster 2024-03-07T03:38:26.987728+0000 mgr.y (mgr.34101) 735 : cluster 0 pgmap v1273: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.8 MiB/s wr, 51 op/s 2024-03-07T03:38:29.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:28 smithi148 bash[82222]: cluster 2024-03-07T03:38:26.987728+0000 mgr.y (mgr.34101) 735 : cluster 0 pgmap v1273: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.8 MiB/s wr, 51 op/s 2024-03-07T03:38:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:28 smithi195 bash[66542]: cluster 2024-03-07T03:38:26.987728+0000 mgr.y (mgr.34101) 735 : cluster 0 pgmap v1273: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.8 MiB/s wr, 51 op/s 2024-03-07T03:38:30.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:29 smithi195 bash[66542]: cluster 2024-03-07T03:38:28.988798+0000 mgr.y (mgr.34101) 736 : cluster 0 pgmap v1274: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:38:30.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:29 smithi148 bash[79819]: cluster 2024-03-07T03:38:28.988798+0000 mgr.y (mgr.34101) 736 : cluster 0 pgmap v1274: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:38:30.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:29 smithi148 bash[82222]: cluster 2024-03-07T03:38:28.988798+0000 mgr.y (mgr.34101) 736 : cluster 0 pgmap v1274: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 44 op/s 2024-03-07T03:38:31.298 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/3.ShrinkToOne (15072 ms) 2024-03-07T03:38:31.299 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/3.ShrinkToOneAfterSnapshot 2024-03-07T03:38:31.299 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:38:32.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:32 smithi148 bash[79819]: cluster 2024-03-07T03:38:30.989721+0000 mgr.y (mgr.34101) 737 : cluster 0 pgmap v1275: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 3.2 MiB/s wr, 39 op/s 2024-03-07T03:38:32.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:32 smithi148 bash[82222]: cluster 2024-03-07T03:38:30.989721+0000 mgr.y (mgr.34101) 737 : cluster 0 pgmap v1275: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 3.2 MiB/s wr, 39 op/s 2024-03-07T03:38:32.304 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:32 smithi195 bash[66542]: cluster 2024-03-07T03:38:30.989721+0000 mgr.y (mgr.34101) 737 : cluster 0 pgmap v1275: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 3.2 MiB/s wr, 39 op/s 2024-03-07T03:38:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:38:32 smithi195 bash[59134]: ts=2024-03-07T03:38:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:34.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:34 smithi148 bash[79819]: cluster 2024-03-07T03:38:32.990479+0000 mgr.y (mgr.34101) 738 : cluster 0 pgmap v1276: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 438 KiB/s rd, 3.1 MiB/s wr, 35 op/s 2024-03-07T03:38:34.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:34 smithi148 bash[82222]: cluster 2024-03-07T03:38:32.990479+0000 mgr.y (mgr.34101) 738 : cluster 0 pgmap v1276: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 438 KiB/s rd, 3.1 MiB/s wr, 35 op/s 2024-03-07T03:38:34.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:34 smithi195 bash[66542]: cluster 2024-03-07T03:38:32.990479+0000 mgr.y (mgr.34101) 738 : cluster 0 pgmap v1276: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 438 KiB/s rd, 3.1 MiB/s wr, 35 op/s 2024-03-07T03:38:36.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:36 smithi195 bash[66542]: cluster 2024-03-07T03:38:34.991731+0000 mgr.y (mgr.34101) 739 : cluster 0 pgmap v1277: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 784 KiB/s rd, 3.4 MiB/s wr, 40 op/s 2024-03-07T03:38:36.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:36 smithi195 bash[66542]: cluster 2024-03-07T03:38:35.069200+0000 mon.a (mon.0) 1598 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2024-03-07T03:38:36.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:36 smithi148 bash[82222]: cluster 2024-03-07T03:38:34.991731+0000 mgr.y (mgr.34101) 739 : cluster 0 pgmap v1277: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 784 KiB/s rd, 3.4 MiB/s wr, 40 op/s 2024-03-07T03:38:36.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:36 smithi148 bash[82222]: cluster 2024-03-07T03:38:35.069200+0000 mon.a (mon.0) 1598 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2024-03-07T03:38:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:36 smithi148 bash[79819]: cluster 2024-03-07T03:38:34.991731+0000 mgr.y (mgr.34101) 739 : cluster 0 pgmap v1277: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 784 KiB/s rd, 3.4 MiB/s wr, 40 op/s 2024-03-07T03:38:36.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:36 smithi148 bash[79819]: cluster 2024-03-07T03:38:35.069200+0000 mon.a (mon.0) 1598 : cluster 0 osdmap e920: 8 total, 8 up, 8 in 2024-03-07T03:38:37.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:38:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:38:37] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:38:38.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:38 smithi195 bash[66542]: cluster 2024-03-07T03:38:36.992710+0000 mgr.y (mgr.34101) 740 : cluster 0 pgmap v1279: 105 pgs: 105 active+clean; 225 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 895 KiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-03-07T03:38:38.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:38 smithi148 bash[82222]: cluster 2024-03-07T03:38:36.992710+0000 mgr.y (mgr.34101) 740 : cluster 0 pgmap v1279: 105 pgs: 105 active+clean; 225 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 895 KiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-03-07T03:38:38.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:38 smithi148 bash[79819]: cluster 2024-03-07T03:38:36.992710+0000 mgr.y (mgr.34101) 740 : cluster 0 pgmap v1279: 105 pgs: 105 active+clean; 225 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 895 KiB/s rd, 2.5 MiB/s wr, 29 op/s 2024-03-07T03:38:40.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:40 smithi195 bash[66542]: cluster 2024-03-07T03:38:38.993755+0000 mgr.y (mgr.34101) 741 : cluster 0 pgmap v1280: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 870 KiB/s rd, 4.1 MiB/s wr, 32 op/s 2024-03-07T03:38:40.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:40 smithi148 bash[79819]: cluster 2024-03-07T03:38:38.993755+0000 mgr.y (mgr.34101) 741 : cluster 0 pgmap v1280: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 870 KiB/s rd, 4.1 MiB/s wr, 32 op/s 2024-03-07T03:38:40.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:40 smithi148 bash[82222]: cluster 2024-03-07T03:38:38.993755+0000 mgr.y (mgr.34101) 741 : cluster 0 pgmap v1280: 105 pgs: 105 active+clean; 241 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 870 KiB/s rd, 4.1 MiB/s wr, 32 op/s 2024-03-07T03:38:42.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:42 smithi195 bash[66542]: cluster 2024-03-07T03:38:40.994385+0000 mgr.y (mgr.34101) 742 : cluster 0 pgmap v1281: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 886 KiB/s rd, 4.1 MiB/s wr, 52 op/s 2024-03-07T03:38:42.314 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:38:42 smithi195 bash[59134]: ts=2024-03-07T03:38:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:42.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:42 smithi148 bash[79819]: cluster 2024-03-07T03:38:40.994385+0000 mgr.y (mgr.34101) 742 : cluster 0 pgmap v1281: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 886 KiB/s rd, 4.1 MiB/s wr, 52 op/s 2024-03-07T03:38:42.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:42 smithi148 bash[82222]: cluster 2024-03-07T03:38:40.994385+0000 mgr.y (mgr.34101) 742 : cluster 0 pgmap v1281: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 886 KiB/s rd, 4.1 MiB/s wr, 52 op/s 2024-03-07T03:38:43.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:43 smithi148 bash[82222]: audit 2024-03-07T03:38:42.335968+0000 mon.a (mon.0) 1599 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:43.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:43 smithi148 bash[79819]: audit 2024-03-07T03:38:42.335968+0000 mon.a (mon.0) 1599 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:43.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:43 smithi195 bash[66542]: audit 2024-03-07T03:38:42.335968+0000 mon.a (mon.0) 1599 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:44.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:44 smithi148 bash[82222]: cluster 2024-03-07T03:38:42.995096+0000 mgr.y (mgr.34101) 743 : cluster 0 pgmap v1282: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 476 KiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:38:44.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:44 smithi148 bash[82222]: cluster 2024-03-07T03:38:43.077869+0000 mon.a (mon.0) 1600 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2024-03-07T03:38:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:44 smithi148 bash[79819]: cluster 2024-03-07T03:38:42.995096+0000 mgr.y (mgr.34101) 743 : cluster 0 pgmap v1282: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 476 KiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:38:44.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:44 smithi148 bash[79819]: cluster 2024-03-07T03:38:43.077869+0000 mon.a (mon.0) 1600 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2024-03-07T03:38:44.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:44 smithi195 bash[66542]: cluster 2024-03-07T03:38:42.995096+0000 mgr.y (mgr.34101) 743 : cluster 0 pgmap v1282: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 476 KiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:38:44.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:44 smithi195 bash[66542]: cluster 2024-03-07T03:38:43.077869+0000 mon.a (mon.0) 1600 : cluster 0 osdmap e921: 8 total, 8 up, 8 in 2024-03-07T03:38:45.555 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/3.ShrinkToOneAfterSnapshot (14257 ms) 2024-03-07T03:38:45.555 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/3.MinOverlap 2024-03-07T03:38:45.555 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:38:46.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:46 smithi148 bash[79819]: cluster 2024-03-07T03:38:44.996354+0000 mgr.y (mgr.34101) 744 : cluster 0 pgmap v1284: 105 pgs: 105 active+clean; 290 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 6.7 MiB/s wr, 69 op/s 2024-03-07T03:38:46.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:46 smithi148 bash[82222]: cluster 2024-03-07T03:38:44.996354+0000 mgr.y (mgr.34101) 744 : cluster 0 pgmap v1284: 105 pgs: 105 active+clean; 290 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 6.7 MiB/s wr, 69 op/s 2024-03-07T03:38:46.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:46 smithi195 bash[66542]: cluster 2024-03-07T03:38:44.996354+0000 mgr.y (mgr.34101) 744 : cluster 0 pgmap v1284: 105 pgs: 105 active+clean; 290 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 6.7 MiB/s wr, 69 op/s 2024-03-07T03:38:47.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:38:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:38:47] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:38:48.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:48 smithi148 bash[82222]: cluster 2024-03-07T03:38:46.996923+0000 mgr.y (mgr.34101) 745 : cluster 0 pgmap v1285: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.7 MiB/s wr, 63 op/s 2024-03-07T03:38:48.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:48 smithi148 bash[79819]: cluster 2024-03-07T03:38:46.996923+0000 mgr.y (mgr.34101) 745 : cluster 0 pgmap v1285: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.7 MiB/s wr, 63 op/s 2024-03-07T03:38:48.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:48 smithi195 bash[66542]: cluster 2024-03-07T03:38:46.996923+0000 mgr.y (mgr.34101) 745 : cluster 0 pgmap v1285: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 7.7 MiB/s wr, 63 op/s 2024-03-07T03:38:50.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:50 smithi148 bash[79819]: cluster 2024-03-07T03:38:48.997960+0000 mgr.y (mgr.34101) 746 : cluster 0 pgmap v1286: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.1 MiB/s wr, 57 op/s 2024-03-07T03:38:50.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:50 smithi148 bash[82222]: cluster 2024-03-07T03:38:48.997960+0000 mgr.y (mgr.34101) 746 : cluster 0 pgmap v1286: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.1 MiB/s wr, 57 op/s 2024-03-07T03:38:50.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:50 smithi195 bash[66542]: cluster 2024-03-07T03:38:48.997960+0000 mgr.y (mgr.34101) 746 : cluster 0 pgmap v1286: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.1 MiB/s wr, 57 op/s 2024-03-07T03:38:51.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:51 smithi148 bash[79819]: cluster 2024-03-07T03:38:50.108305+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2024-03-07T03:38:51.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:51 smithi148 bash[82222]: cluster 2024-03-07T03:38:50.108305+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2024-03-07T03:38:51.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:51 smithi195 bash[66542]: cluster 2024-03-07T03:38:50.108305+0000 mon.a (mon.0) 1601 : cluster 0 osdmap e922: 8 total, 8 up, 8 in 2024-03-07T03:38:52.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:52 smithi148 bash[79819]: cluster 2024-03-07T03:38:50.998470+0000 mgr.y (mgr.34101) 747 : cluster 0 pgmap v1288: 105 pgs: 105 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 8.1 MiB/s wr, 51 op/s 2024-03-07T03:38:52.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:52 smithi148 bash[82222]: cluster 2024-03-07T03:38:50.998470+0000 mgr.y (mgr.34101) 747 : cluster 0 pgmap v1288: 105 pgs: 105 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 8.1 MiB/s wr, 51 op/s 2024-03-07T03:38:52.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:52 smithi195 bash[66542]: cluster 2024-03-07T03:38:50.998470+0000 mgr.y (mgr.34101) 747 : cluster 0 pgmap v1288: 105 pgs: 105 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 8.1 MiB/s wr, 51 op/s 2024-03-07T03:38:52.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:38:52 smithi195 bash[59134]: ts=2024-03-07T03:38:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:38:54.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:54 smithi148 bash[82222]: cluster 2024-03-07T03:38:52.999188+0000 mgr.y (mgr.34101) 748 : cluster 0 pgmap v1289: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.6 MiB/s wr, 54 op/s 2024-03-07T03:38:54.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:54 smithi148 bash[79819]: cluster 2024-03-07T03:38:52.999188+0000 mgr.y (mgr.34101) 748 : cluster 0 pgmap v1289: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.6 MiB/s wr, 54 op/s 2024-03-07T03:38:54.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:54 smithi195 bash[66542]: cluster 2024-03-07T03:38:52.999188+0000 mgr.y (mgr.34101) 748 : cluster 0 pgmap v1289: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.6 MiB/s wr, 54 op/s 2024-03-07T03:38:56.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:56 smithi148 bash[79819]: cluster 2024-03-07T03:38:55.000346+0000 mgr.y (mgr.34101) 749 : cluster 0 pgmap v1290: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:38:56.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:56 smithi148 bash[82222]: cluster 2024-03-07T03:38:55.000346+0000 mgr.y (mgr.34101) 749 : cluster 0 pgmap v1290: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:38:56.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:56 smithi195 bash[66542]: cluster 2024-03-07T03:38:55.000346+0000 mgr.y (mgr.34101) 749 : cluster 0 pgmap v1290: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:38:57.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:38:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:38:57] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:38:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:58 smithi148 bash[79819]: cluster 2024-03-07T03:38:57.001205+0000 mgr.y (mgr.34101) 750 : cluster 0 pgmap v1291: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 44 op/s 2024-03-07T03:38:58.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:38:58 smithi148 bash[79819]: audit 2024-03-07T03:38:57.336739+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:58 smithi148 bash[82222]: cluster 2024-03-07T03:38:57.001205+0000 mgr.y (mgr.34101) 750 : cluster 0 pgmap v1291: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 44 op/s 2024-03-07T03:38:58.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:38:58 smithi148 bash[82222]: audit 2024-03-07T03:38:57.336739+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:38:58.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:58 smithi195 bash[66542]: cluster 2024-03-07T03:38:57.001205+0000 mgr.y (mgr.34101) 750 : cluster 0 pgmap v1291: 105 pgs: 105 active+clean; 331 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 44 op/s 2024-03-07T03:38:58.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:38:58 smithi195 bash[66542]: audit 2024-03-07T03:38:57.336739+0000 mon.a (mon.0) 1602 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:00.539 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:00 smithi148 bash[79819]: cluster 2024-03-07T03:38:59.002213+0000 mgr.y (mgr.34101) 751 : cluster 0 pgmap v1292: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 3.3 MiB/s wr, 47 op/s 2024-03-07T03:39:00.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:00 smithi148 bash[82222]: cluster 2024-03-07T03:38:59.002213+0000 mgr.y (mgr.34101) 751 : cluster 0 pgmap v1292: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 3.3 MiB/s wr, 47 op/s 2024-03-07T03:39:00.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:00 smithi195 bash[66542]: cluster 2024-03-07T03:38:59.002213+0000 mgr.y (mgr.34101) 751 : cluster 0 pgmap v1292: 105 pgs: 105 active+clean; 335 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 3.3 MiB/s wr, 47 op/s 2024-03-07T03:39:00.790 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/3.MinOverlap (15235 ms) 2024-03-07T03:39:00.790 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/3.ZeroOverlap 2024-03-07T03:39:00.790 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:39:02.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:02 smithi148 bash[82222]: cluster 2024-03-07T03:39:01.003069+0000 mgr.y (mgr.34101) 752 : cluster 0 pgmap v1293: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 833 KiB/s rd, 3.0 MiB/s wr, 39 op/s 2024-03-07T03:39:02.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:02 smithi148 bash[79819]: cluster 2024-03-07T03:39:01.003069+0000 mgr.y (mgr.34101) 752 : cluster 0 pgmap v1293: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 833 KiB/s rd, 3.0 MiB/s wr, 39 op/s 2024-03-07T03:39:02.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:02 smithi195 bash[66542]: cluster 2024-03-07T03:39:01.003069+0000 mgr.y (mgr.34101) 752 : cluster 0 pgmap v1293: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 833 KiB/s rd, 3.0 MiB/s wr, 39 op/s 2024-03-07T03:39:02.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:39:02 smithi195 bash[59134]: ts=2024-03-07T03:39:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:04.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:04 smithi148 bash[79819]: cluster 2024-03-07T03:39:03.003773+0000 mgr.y (mgr.34101) 753 : cluster 0 pgmap v1294: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 764 KiB/s rd, 2.8 MiB/s wr, 45 op/s 2024-03-07T03:39:04.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:04 smithi148 bash[79819]: audit 2024-03-07T03:39:03.628010+0000 mon.a (mon.0) 1603 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:39:04.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:04 smithi148 bash[82222]: cluster 2024-03-07T03:39:03.003773+0000 mgr.y (mgr.34101) 753 : cluster 0 pgmap v1294: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 764 KiB/s rd, 2.8 MiB/s wr, 45 op/s 2024-03-07T03:39:04.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:04 smithi148 bash[82222]: audit 2024-03-07T03:39:03.628010+0000 mon.a (mon.0) 1603 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:39:04.563 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:04 smithi195 bash[66542]: cluster 2024-03-07T03:39:03.003773+0000 mgr.y (mgr.34101) 753 : cluster 0 pgmap v1294: 105 pgs: 105 active+clean; 339 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 764 KiB/s rd, 2.8 MiB/s wr, 45 op/s 2024-03-07T03:39:04.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:04 smithi195 bash[66542]: audit 2024-03-07T03:39:03.628010+0000 mon.a (mon.0) 1603 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:39:06.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:06 smithi148 bash[79819]: cluster 2024-03-07T03:39:05.004967+0000 mgr.y (mgr.34101) 754 : cluster 0 pgmap v1295: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.1 MiB/s wr, 38 op/s 2024-03-07T03:39:06.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:06 smithi148 bash[79819]: cluster 2024-03-07T03:39:05.160998+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2024-03-07T03:39:06.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:06 smithi148 bash[82222]: cluster 2024-03-07T03:39:05.004967+0000 mgr.y (mgr.34101) 754 : cluster 0 pgmap v1295: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.1 MiB/s wr, 38 op/s 2024-03-07T03:39:06.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:06 smithi148 bash[82222]: cluster 2024-03-07T03:39:05.160998+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2024-03-07T03:39:06.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:06 smithi195 bash[66542]: cluster 2024-03-07T03:39:05.004967+0000 mgr.y (mgr.34101) 754 : cluster 0 pgmap v1295: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.1 MiB/s wr, 38 op/s 2024-03-07T03:39:06.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:06 smithi195 bash[66542]: cluster 2024-03-07T03:39:05.160998+0000 mon.a (mon.0) 1604 : cluster 0 osdmap e923: 8 total, 8 up, 8 in 2024-03-07T03:39:07.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:39:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:39:07] "GET /metrics HTTP/1.1" 200 36735 "" "Prometheus/2.43.0" 2024-03-07T03:39:08.539 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:08 smithi148 bash[82222]: cluster 2024-03-07T03:39:07.005813+0000 mgr.y (mgr.34101) 755 : cluster 0 pgmap v1297: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 36 op/s 2024-03-07T03:39:08.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:08 smithi148 bash[79819]: cluster 2024-03-07T03:39:07.005813+0000 mgr.y (mgr.34101) 755 : cluster 0 pgmap v1297: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 36 op/s 2024-03-07T03:39:08.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:08 smithi195 bash[66542]: cluster 2024-03-07T03:39:07.005813+0000 mgr.y (mgr.34101) 755 : cluster 0 pgmap v1297: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.1 MiB/s wr, 36 op/s 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: cluster 2024-03-07T03:39:09.006438+0000 mgr.y (mgr.34101) 756 : cluster 0 pgmap v1298: 105 pgs: 105 active+clean; 360 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: audit 2024-03-07T03:39:09.419346+0000 mon.a (mon.0) 1605 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: audit 2024-03-07T03:39:09.428560+0000 mon.a (mon.0) 1606 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: audit 2024-03-07T03:39:09.726222+0000 mon.a (mon.0) 1607 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: audit 2024-03-07T03:39:09.737250+0000 mon.a (mon.0) 1608 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: audit 2024-03-07T03:39:10.325063+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: audit 2024-03-07T03:39:10.326558+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:39:10.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[79819]: audit 2024-03-07T03:39:10.333150+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: cluster 2024-03-07T03:39:09.006438+0000 mgr.y (mgr.34101) 756 : cluster 0 pgmap v1298: 105 pgs: 105 active+clean; 360 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: audit 2024-03-07T03:39:09.419346+0000 mon.a (mon.0) 1605 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: audit 2024-03-07T03:39:09.428560+0000 mon.a (mon.0) 1606 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: audit 2024-03-07T03:39:09.726222+0000 mon.a (mon.0) 1607 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: audit 2024-03-07T03:39:09.737250+0000 mon.a (mon.0) 1608 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: audit 2024-03-07T03:39:10.325063+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: audit 2024-03-07T03:39:10.326558+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:39:10.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:10 smithi148 bash[82222]: audit 2024-03-07T03:39:10.333150+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: cluster 2024-03-07T03:39:09.006438+0000 mgr.y (mgr.34101) 756 : cluster 0 pgmap v1298: 105 pgs: 105 active+clean; 360 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: audit 2024-03-07T03:39:09.419346+0000 mon.a (mon.0) 1605 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: audit 2024-03-07T03:39:09.428560+0000 mon.a (mon.0) 1606 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: audit 2024-03-07T03:39:09.726222+0000 mon.a (mon.0) 1607 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: audit 2024-03-07T03:39:09.737250+0000 mon.a (mon.0) 1608 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: audit 2024-03-07T03:39:10.325063+0000 mon.a (mon.0) 1609 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: audit 2024-03-07T03:39:10.326558+0000 mon.a (mon.0) 1610 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:39:10.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:10 smithi195 bash[66542]: audit 2024-03-07T03:39:10.333150+0000 mon.a (mon.0) 1611 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:12 smithi195 bash[66542]: cluster 2024-03-07T03:39:11.007306+0000 mgr.y (mgr.34101) 757 : cluster 0 pgmap v1299: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 50 op/s 2024-03-07T03:39:12.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:12 smithi195 bash[66542]: audit 2024-03-07T03:39:12.336723+0000 mon.a (mon.0) 1612 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:39:12 smithi195 bash[59134]: ts=2024-03-07T03:39:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:12 smithi148 bash[79819]: cluster 2024-03-07T03:39:11.007306+0000 mgr.y (mgr.34101) 757 : cluster 0 pgmap v1299: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 50 op/s 2024-03-07T03:39:12.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:12 smithi148 bash[79819]: audit 2024-03-07T03:39:12.336723+0000 mon.a (mon.0) 1612 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:12 smithi148 bash[82222]: cluster 2024-03-07T03:39:11.007306+0000 mgr.y (mgr.34101) 757 : cluster 0 pgmap v1299: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 50 op/s 2024-03-07T03:39:12.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:12 smithi148 bash[82222]: audit 2024-03-07T03:39:12.336723+0000 mon.a (mon.0) 1612 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:14.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:14 smithi148 bash[79819]: cluster 2024-03-07T03:39:13.008065+0000 mgr.y (mgr.34101) 758 : cluster 0 pgmap v1300: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 42 op/s 2024-03-07T03:39:14.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:14 smithi148 bash[82222]: cluster 2024-03-07T03:39:13.008065+0000 mgr.y (mgr.34101) 758 : cluster 0 pgmap v1300: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 42 op/s 2024-03-07T03:39:14.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:14 smithi195 bash[66542]: cluster 2024-03-07T03:39:13.008065+0000 mgr.y (mgr.34101) 758 : cluster 0 pgmap v1300: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 3.7 MiB/s wr, 42 op/s 2024-03-07T03:39:15.574 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/3.ZeroOverlap (14784 ms) 2024-03-07T03:39:16.441 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/3 (108446 ms total) 2024-03-07T03:39:16.441 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:39:16.441 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/4, where TypeParam = LUKS1UnderLUKS2 2024-03-07T03:39:16.482 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/4.Simple 2024-03-07T03:39:16.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:16 smithi148 bash[79819]: cluster 2024-03-07T03:39:15.009175+0000 mgr.y (mgr.34101) 759 : cluster 0 pgmap v1301: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:39:16.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:16 smithi148 bash[82222]: cluster 2024-03-07T03:39:15.009175+0000 mgr.y (mgr.34101) 759 : cluster 0 pgmap v1301: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:39:16.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:16 smithi195 bash[66542]: cluster 2024-03-07T03:39:15.009175+0000 mgr.y (mgr.34101) 759 : cluster 0 pgmap v1301: 105 pgs: 105 active+clean; 372 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-03-07T03:39:17.445 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:39:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:39:17] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.43.0" 2024-03-07T03:39:17.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:17 smithi148 bash[79819]: cluster 2024-03-07T03:39:16.435039+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2024-03-07T03:39:17.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:17 smithi148 bash[82222]: cluster 2024-03-07T03:39:16.435039+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2024-03-07T03:39:17.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:17 smithi195 bash[66542]: cluster 2024-03-07T03:39:16.435039+0000 mon.a (mon.0) 1613 : cluster 0 osdmap e924: 8 total, 8 up, 8 in 2024-03-07T03:39:18.468 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:39:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:18 smithi148 bash[79819]: cluster 2024-03-07T03:39:17.009818+0000 mgr.y (mgr.34101) 760 : cluster 0 pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:18 smithi148 bash[79819]: cluster 2024-03-07T03:39:17.466402+0000 mon.a (mon.0) 1614 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2024-03-07T03:39:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:18 smithi148 bash[79819]: audit 2024-03-07T03:39:17.474764+0000 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.148:0/47256214' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:18 smithi148 bash[82222]: cluster 2024-03-07T03:39:17.009818+0000 mgr.y (mgr.34101) 760 : cluster 0 pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:18 smithi148 bash[82222]: cluster 2024-03-07T03:39:17.466402+0000 mon.a (mon.0) 1614 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2024-03-07T03:39:18.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:18 smithi148 bash[82222]: audit 2024-03-07T03:39:17.474764+0000 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.148:0/47256214' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:18 smithi195 bash[66542]: cluster 2024-03-07T03:39:17.009818+0000 mgr.y (mgr.34101) 760 : cluster 0 pgmap v1303: 73 pgs: 73 active+clean; 582 KiB data, 1.6 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:18 smithi195 bash[66542]: cluster 2024-03-07T03:39:17.466402+0000 mon.a (mon.0) 1614 : cluster 0 osdmap e925: 8 total, 8 up, 8 in 2024-03-07T03:39:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:18 smithi195 bash[66542]: audit 2024-03-07T03:39:17.474764+0000 mon.a (mon.0) 1615 : audit 1 from='client.? 172.21.15.148:0/47256214' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-40","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:39:19.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:19 smithi148 bash[79819]: audit 2024-03-07T03:39:18.465972+0000 mon.a (mon.0) 1616 : audit 1 from='client.? 172.21.15.148:0/47256214' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:39:19.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:19 smithi148 bash[79819]: cluster 2024-03-07T03:39:18.474232+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2024-03-07T03:39:19.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:19 smithi148 bash[82222]: audit 2024-03-07T03:39:18.465972+0000 mon.a (mon.0) 1616 : audit 1 from='client.? 172.21.15.148:0/47256214' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:39:19.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:19 smithi148 bash[82222]: cluster 2024-03-07T03:39:18.474232+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2024-03-07T03:39:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:19 smithi195 bash[66542]: audit 2024-03-07T03:39:18.465972+0000 mon.a (mon.0) 1616 : audit 1 from='client.? 172.21.15.148:0/47256214' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:39:19.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:19 smithi195 bash[66542]: cluster 2024-03-07T03:39:18.474232+0000 mon.a (mon.0) 1617 : cluster 0 osdmap e926: 8 total, 8 up, 8 in 2024-03-07T03:39:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:20 smithi148 bash[79819]: cluster 2024-03-07T03:39:19.010758+0000 mgr.y (mgr.34101) 761 : cluster 0 pgmap v1306: 105 pgs: 7 creating+peering, 16 unknown, 82 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:20 smithi148 bash[79819]: cluster 2024-03-07T03:39:19.484236+0000 mon.a (mon.0) 1618 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2024-03-07T03:39:20.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:20 smithi148 bash[82222]: cluster 2024-03-07T03:39:19.010758+0000 mgr.y (mgr.34101) 761 : cluster 0 pgmap v1306: 105 pgs: 7 creating+peering, 16 unknown, 82 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:20.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:20 smithi148 bash[82222]: cluster 2024-03-07T03:39:19.484236+0000 mon.a (mon.0) 1618 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2024-03-07T03:39:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:20 smithi195 bash[66542]: cluster 2024-03-07T03:39:19.010758+0000 mgr.y (mgr.34101) 761 : cluster 0 pgmap v1306: 105 pgs: 7 creating+peering, 16 unknown, 82 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:39:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:20 smithi195 bash[66542]: cluster 2024-03-07T03:39:19.484236+0000 mon.a (mon.0) 1618 : cluster 0 osdmap e927: 8 total, 8 up, 8 in 2024-03-07T03:39:21.789 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:21 smithi148 bash[82222]: cluster 2024-03-07T03:39:20.492244+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2024-03-07T03:39:21.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:21 smithi148 bash[79819]: cluster 2024-03-07T03:39:20.492244+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2024-03-07T03:39:21.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:21 smithi195 bash[66542]: cluster 2024-03-07T03:39:20.492244+0000 mon.a (mon.0) 1619 : cluster 0 osdmap e928: 8 total, 8 up, 8 in 2024-03-07T03:39:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:22 smithi195 bash[66542]: cluster 2024-03-07T03:39:21.011431+0000 mgr.y (mgr.34101) 762 : cluster 0 pgmap v1309: 105 pgs: 7 creating+peering, 3 unknown, 95 active+clean; 582 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:39:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:39:22 smithi195 bash[59134]: ts=2024-03-07T03:39:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:22.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:22 smithi148 bash[79819]: cluster 2024-03-07T03:39:21.011431+0000 mgr.y (mgr.34101) 762 : cluster 0 pgmap v1309: 105 pgs: 7 creating+peering, 3 unknown, 95 active+clean; 582 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:39:22.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:22 smithi148 bash[82222]: cluster 2024-03-07T03:39:21.011431+0000 mgr.y (mgr.34101) 762 : cluster 0 pgmap v1309: 105 pgs: 7 creating+peering, 3 unknown, 95 active+clean; 582 KiB data, 720 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:39:24.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:24 smithi148 bash[79819]: cluster 2024-03-07T03:39:23.012179+0000 mgr.y (mgr.34101) 763 : cluster 0 pgmap v1310: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 593 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 368 B/s wr, 1 op/s 2024-03-07T03:39:24.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:24 smithi148 bash[82222]: cluster 2024-03-07T03:39:23.012179+0000 mgr.y (mgr.34101) 763 : cluster 0 pgmap v1310: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 593 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 368 B/s wr, 1 op/s 2024-03-07T03:39:24.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:24 smithi195 bash[66542]: cluster 2024-03-07T03:39:23.012179+0000 mgr.y (mgr.34101) 763 : cluster 0 pgmap v1310: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 593 MiB used, 715 GiB / 715 GiB avail; 553 B/s rd, 368 B/s wr, 1 op/s 2024-03-07T03:39:25.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:25 smithi148 bash[79819]: cluster 2024-03-07T03:39:24.517847+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2024-03-07T03:39:25.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:25 smithi148 bash[82222]: cluster 2024-03-07T03:39:24.517847+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2024-03-07T03:39:25.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:25 smithi195 bash[66542]: cluster 2024-03-07T03:39:24.517847+0000 mon.a (mon.0) 1620 : cluster 0 osdmap e929: 8 total, 8 up, 8 in 2024-03-07T03:39:26.789 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:26 smithi148 bash[79819]: cluster 2024-03-07T03:39:25.012913+0000 mgr.y (mgr.34101) 764 : cluster 0 pgmap v1312: 105 pgs: 105 active+clean; 4.6 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 683 KiB/s wr, 3 op/s 2024-03-07T03:39:26.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:26 smithi148 bash[82222]: cluster 2024-03-07T03:39:25.012913+0000 mgr.y (mgr.34101) 764 : cluster 0 pgmap v1312: 105 pgs: 105 active+clean; 4.6 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 683 KiB/s wr, 3 op/s 2024-03-07T03:39:26.813 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:26 smithi195 bash[66542]: cluster 2024-03-07T03:39:25.012913+0000 mgr.y (mgr.34101) 764 : cluster 0 pgmap v1312: 105 pgs: 105 active+clean; 4.6 MiB data, 609 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 683 KiB/s wr, 3 op/s 2024-03-07T03:39:27.516 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:27 smithi148 bash[79819]: audit 2024-03-07T03:39:27.336815+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:27.517 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:27 smithi148 bash[82222]: audit 2024-03-07T03:39:27.336815+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:27.517 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:39:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:39:27] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.43.0" 2024-03-07T03:39:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:27 smithi195 bash[66542]: audit 2024-03-07T03:39:27.336815+0000 mon.a (mon.0) 1621 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:29.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:28 smithi148 bash[79819]: cluster 2024-03-07T03:39:27.013634+0000 mgr.y (mgr.34101) 765 : cluster 0 pgmap v1313: 105 pgs: 105 active+clean; 8.6 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 1.1 MiB/s wr, 48 op/s 2024-03-07T03:39:29.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:28 smithi148 bash[79819]: cluster 2024-03-07T03:39:27.610921+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2024-03-07T03:39:29.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:28 smithi148 bash[82222]: cluster 2024-03-07T03:39:27.013634+0000 mgr.y (mgr.34101) 765 : cluster 0 pgmap v1313: 105 pgs: 105 active+clean; 8.6 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 1.1 MiB/s wr, 48 op/s 2024-03-07T03:39:29.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:28 smithi148 bash[82222]: cluster 2024-03-07T03:39:27.610921+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2024-03-07T03:39:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:28 smithi195 bash[66542]: cluster 2024-03-07T03:39:27.013634+0000 mgr.y (mgr.34101) 765 : cluster 0 pgmap v1313: 105 pgs: 105 active+clean; 8.6 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 1.1 MiB/s wr, 48 op/s 2024-03-07T03:39:29.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:28 smithi195 bash[66542]: cluster 2024-03-07T03:39:27.610921+0000 mon.a (mon.0) 1622 : cluster 0 osdmap e930: 8 total, 8 up, 8 in 2024-03-07T03:39:30.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:29 smithi148 bash[79819]: audit 2024-03-07T03:39:28.660412+0000 mon.a (mon.0) 1623 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:30.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:29 smithi148 bash[82222]: audit 2024-03-07T03:39:28.660412+0000 mon.a (mon.0) 1623 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:30.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:29 smithi195 bash[66542]: audit 2024-03-07T03:39:28.660412+0000 mon.a (mon.0) 1623 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:39:31.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:30 smithi148 bash[79819]: cluster 2024-03-07T03:39:29.014355+0000 mgr.y (mgr.34101) 766 : cluster 0 pgmap v1315: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 3.6 MiB/s wr, 48 op/s 2024-03-07T03:39:31.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:30 smithi148 bash[82222]: cluster 2024-03-07T03:39:29.014355+0000 mgr.y (mgr.34101) 766 : cluster 0 pgmap v1315: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 3.6 MiB/s wr, 48 op/s 2024-03-07T03:39:31.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:30 smithi195 bash[66542]: cluster 2024-03-07T03:39:29.014355+0000 mgr.y (mgr.34101) 766 : cluster 0 pgmap v1315: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 3.6 MiB/s wr, 48 op/s 2024-03-07T03:39:32.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:39:32 smithi195 bash[59134]: ts=2024-03-07T03:39:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:33.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:32 smithi148 bash[79819]: cluster 2024-03-07T03:39:31.015005+0000 mgr.y (mgr.34101) 767 : cluster 0 pgmap v1316: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 3.6 MiB/s wr, 49 op/s 2024-03-07T03:39:33.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:32 smithi148 bash[82222]: cluster 2024-03-07T03:39:31.015005+0000 mgr.y (mgr.34101) 767 : cluster 0 pgmap v1316: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 3.6 MiB/s wr, 49 op/s 2024-03-07T03:39:33.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:32 smithi195 bash[66542]: cluster 2024-03-07T03:39:31.015005+0000 mgr.y (mgr.34101) 767 : cluster 0 pgmap v1316: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 3.6 MiB/s wr, 49 op/s 2024-03-07T03:39:35.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:34 smithi148 bash[79819]: cluster 2024-03-07T03:39:33.015579+0000 mgr.y (mgr.34101) 768 : cluster 0 pgmap v1317: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 44 op/s 2024-03-07T03:39:35.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:34 smithi148 bash[82222]: cluster 2024-03-07T03:39:33.015579+0000 mgr.y (mgr.34101) 768 : cluster 0 pgmap v1317: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 44 op/s 2024-03-07T03:39:35.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:34 smithi195 bash[66542]: cluster 2024-03-07T03:39:33.015579+0000 mgr.y (mgr.34101) 768 : cluster 0 pgmap v1317: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.9 MiB/s wr, 44 op/s 2024-03-07T03:39:37.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:36 smithi148 bash[82222]: cluster 2024-03-07T03:39:35.016735+0000 mgr.y (mgr.34101) 769 : cluster 0 pgmap v1318: 105 pgs: 105 active+clean; 42 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 3.7 MiB/s wr, 42 op/s 2024-03-07T03:39:37.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:36 smithi148 bash[79819]: cluster 2024-03-07T03:39:35.016735+0000 mgr.y (mgr.34101) 769 : cluster 0 pgmap v1318: 105 pgs: 105 active+clean; 42 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 3.7 MiB/s wr, 42 op/s 2024-03-07T03:39:37.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:36 smithi195 bash[66542]: cluster 2024-03-07T03:39:35.016735+0000 mgr.y (mgr.34101) 769 : cluster 0 pgmap v1318: 105 pgs: 105 active+clean; 42 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 3.7 MiB/s wr, 42 op/s 2024-03-07T03:39:37.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:39:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:39:37] "GET /metrics HTTP/1.1" 200 36716 "" "Prometheus/2.43.0" 2024-03-07T03:39:38.043 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/4.Simple (21562 ms) 2024-03-07T03:39:38.043 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/4.Grow 2024-03-07T03:39:38.044 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:39:39.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:38 smithi148 bash[82222]: cluster 2024-03-07T03:39:37.017390+0000 mgr.y (mgr.34101) 770 : cluster 0 pgmap v1319: 105 pgs: 105 active+clean; 54 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 857 KiB/s rd, 4.5 MiB/s wr, 15 op/s 2024-03-07T03:39:39.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:38 smithi148 bash[79819]: cluster 2024-03-07T03:39:37.017390+0000 mgr.y (mgr.34101) 770 : cluster 0 pgmap v1319: 105 pgs: 105 active+clean; 54 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 857 KiB/s rd, 4.5 MiB/s wr, 15 op/s 2024-03-07T03:39:39.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:38 smithi195 bash[66542]: cluster 2024-03-07T03:39:37.017390+0000 mgr.y (mgr.34101) 770 : cluster 0 pgmap v1319: 105 pgs: 105 active+clean; 54 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 857 KiB/s rd, 4.5 MiB/s wr, 15 op/s 2024-03-07T03:39:41.039 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:40 smithi148 bash[82222]: cluster 2024-03-07T03:39:39.018436+0000 mgr.y (mgr.34101) 771 : cluster 0 pgmap v1320: 105 pgs: 105 active+clean; 62 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 5.7 MiB/s wr, 23 op/s 2024-03-07T03:39:41.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:40 smithi148 bash[79819]: cluster 2024-03-07T03:39:39.018436+0000 mgr.y (mgr.34101) 771 : cluster 0 pgmap v1320: 105 pgs: 105 active+clean; 62 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 5.7 MiB/s wr, 23 op/s 2024-03-07T03:39:41.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:40 smithi195 bash[66542]: cluster 2024-03-07T03:39:39.018436+0000 mgr.y (mgr.34101) 771 : cluster 0 pgmap v1320: 105 pgs: 105 active+clean; 62 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 6.9 MiB/s rd, 5.7 MiB/s wr, 23 op/s 2024-03-07T03:39:42.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:39:42 smithi195 bash[59134]: ts=2024-03-07T03:39:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:43.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:42 smithi148 bash[79819]: cluster 2024-03-07T03:39:41.019246+0000 mgr.y (mgr.34101) 772 : cluster 0 pgmap v1321: 105 pgs: 105 active+clean; 71 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 4.8 MiB/s wr, 29 op/s 2024-03-07T03:39:43.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:42 smithi148 bash[79819]: audit 2024-03-07T03:39:42.337897+0000 mon.a (mon.0) 1624 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:42 smithi148 bash[82222]: cluster 2024-03-07T03:39:41.019246+0000 mgr.y (mgr.34101) 772 : cluster 0 pgmap v1321: 105 pgs: 105 active+clean; 71 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 4.8 MiB/s wr, 29 op/s 2024-03-07T03:39:43.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:42 smithi148 bash[82222]: audit 2024-03-07T03:39:42.337897+0000 mon.a (mon.0) 1624 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:42 smithi195 bash[66542]: cluster 2024-03-07T03:39:41.019246+0000 mgr.y (mgr.34101) 772 : cluster 0 pgmap v1321: 105 pgs: 105 active+clean; 71 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 4.8 MiB/s wr, 29 op/s 2024-03-07T03:39:43.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:42 smithi195 bash[66542]: audit 2024-03-07T03:39:42.337897+0000 mon.a (mon.0) 1624 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:44.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:43 smithi148 bash[79819]: cluster 2024-03-07T03:39:42.705131+0000 mon.a (mon.0) 1625 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2024-03-07T03:39:44.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:43 smithi148 bash[82222]: cluster 2024-03-07T03:39:42.705131+0000 mon.a (mon.0) 1625 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2024-03-07T03:39:44.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:43 smithi195 bash[66542]: cluster 2024-03-07T03:39:42.705131+0000 mon.a (mon.0) 1625 : cluster 0 osdmap e931: 8 total, 8 up, 8 in 2024-03-07T03:39:45.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:44 smithi148 bash[79819]: cluster 2024-03-07T03:39:43.019899+0000 mgr.y (mgr.34101) 773 : cluster 0 pgmap v1323: 105 pgs: 105 active+clean; 71 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 34 op/s 2024-03-07T03:39:45.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:44 smithi148 bash[82222]: cluster 2024-03-07T03:39:43.019899+0000 mgr.y (mgr.34101) 773 : cluster 0 pgmap v1323: 105 pgs: 105 active+clean; 71 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 34 op/s 2024-03-07T03:39:45.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:44 smithi195 bash[66542]: cluster 2024-03-07T03:39:43.019899+0000 mgr.y (mgr.34101) 773 : cluster 0 pgmap v1323: 105 pgs: 105 active+clean; 71 MiB data, 803 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 34 op/s 2024-03-07T03:39:47.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:46 smithi148 bash[79819]: cluster 2024-03-07T03:39:45.020721+0000 mgr.y (mgr.34101) 774 : cluster 0 pgmap v1324: 105 pgs: 105 active+clean; 88 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 41 op/s 2024-03-07T03:39:47.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:46 smithi148 bash[82222]: cluster 2024-03-07T03:39:45.020721+0000 mgr.y (mgr.34101) 774 : cluster 0 pgmap v1324: 105 pgs: 105 active+clean; 88 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 41 op/s 2024-03-07T03:39:47.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:46 smithi195 bash[66542]: cluster 2024-03-07T03:39:45.020721+0000 mgr.y (mgr.34101) 774 : cluster 0 pgmap v1324: 105 pgs: 105 active+clean; 88 MiB data, 861 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 41 op/s 2024-03-07T03:39:47.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:39:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:39:47] "GET /metrics HTTP/1.1" 200 36727 "" "Prometheus/2.43.0" 2024-03-07T03:39:49.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:48 smithi148 bash[79819]: cluster 2024-03-07T03:39:47.021351+0000 mgr.y (mgr.34101) 775 : cluster 0 pgmap v1325: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-03-07T03:39:49.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:48 smithi148 bash[82222]: cluster 2024-03-07T03:39:47.021351+0000 mgr.y (mgr.34101) 775 : cluster 0 pgmap v1325: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-03-07T03:39:49.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:48 smithi195 bash[66542]: cluster 2024-03-07T03:39:47.021351+0000 mgr.y (mgr.34101) 775 : cluster 0 pgmap v1325: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-03-07T03:39:51.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:50 smithi148 bash[79819]: cluster 2024-03-07T03:39:49.022113+0000 mgr.y (mgr.34101) 776 : cluster 0 pgmap v1326: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 4.2 MiB/s wr, 45 op/s 2024-03-07T03:39:51.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:50 smithi148 bash[82222]: cluster 2024-03-07T03:39:49.022113+0000 mgr.y (mgr.34101) 776 : cluster 0 pgmap v1326: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 4.2 MiB/s wr, 45 op/s 2024-03-07T03:39:51.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:50 smithi195 bash[66542]: cluster 2024-03-07T03:39:49.022113+0000 mgr.y (mgr.34101) 776 : cluster 0 pgmap v1326: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 4.2 MiB/s wr, 45 op/s 2024-03-07T03:39:52.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:39:52 smithi195 bash[59134]: ts=2024-03-07T03:39:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:39:53.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:52 smithi148 bash[79819]: cluster 2024-03-07T03:39:51.022624+0000 mgr.y (mgr.34101) 777 : cluster 0 pgmap v1327: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 852 KiB/s rd, 2.9 MiB/s wr, 45 op/s 2024-03-07T03:39:53.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:52 smithi148 bash[82222]: cluster 2024-03-07T03:39:51.022624+0000 mgr.y (mgr.34101) 777 : cluster 0 pgmap v1327: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 852 KiB/s rd, 2.9 MiB/s wr, 45 op/s 2024-03-07T03:39:53.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:52 smithi195 bash[66542]: cluster 2024-03-07T03:39:51.022624+0000 mgr.y (mgr.34101) 777 : cluster 0 pgmap v1327: 105 pgs: 105 active+clean; 100 MiB data, 894 MiB used, 714 GiB / 715 GiB avail; 852 KiB/s rd, 2.9 MiB/s wr, 45 op/s 2024-03-07T03:39:55.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:54 smithi148 bash[79819]: cluster 2024-03-07T03:39:53.023065+0000 mgr.y (mgr.34101) 778 : cluster 0 pgmap v1328: 105 pgs: 105 active+clean; 100 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 2.8 MiB/s wr, 43 op/s 2024-03-07T03:39:55.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:54 smithi148 bash[82222]: cluster 2024-03-07T03:39:53.023065+0000 mgr.y (mgr.34101) 778 : cluster 0 pgmap v1328: 105 pgs: 105 active+clean; 100 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 2.8 MiB/s wr, 43 op/s 2024-03-07T03:39:55.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:54 smithi195 bash[66542]: cluster 2024-03-07T03:39:53.023065+0000 mgr.y (mgr.34101) 778 : cluster 0 pgmap v1328: 105 pgs: 105 active+clean; 100 MiB data, 906 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 2.8 MiB/s wr, 43 op/s 2024-03-07T03:39:55.977 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/4.Grow (17933 ms) 2024-03-07T03:39:55.977 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/4.Shrink 2024-03-07T03:39:55.977 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:39:57.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:56 smithi148 bash[79819]: cluster 2024-03-07T03:39:55.023845+0000 mgr.y (mgr.34101) 779 : cluster 0 pgmap v1329: 105 pgs: 105 active+clean; 116 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 759 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-03-07T03:39:57.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:56 smithi148 bash[82222]: cluster 2024-03-07T03:39:55.023845+0000 mgr.y (mgr.34101) 779 : cluster 0 pgmap v1329: 105 pgs: 105 active+clean; 116 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 759 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-03-07T03:39:57.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:56 smithi195 bash[66542]: cluster 2024-03-07T03:39:55.023845+0000 mgr.y (mgr.34101) 779 : cluster 0 pgmap v1329: 105 pgs: 105 active+clean; 116 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 759 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-03-07T03:39:57.539 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:39:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:39:57] "GET /metrics HTTP/1.1" 200 36727 "" "Prometheus/2.43.0" 2024-03-07T03:39:58.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:57 smithi148 bash[79819]: audit 2024-03-07T03:39:57.338096+0000 mon.a (mon.0) 1626 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:58.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:57 smithi148 bash[82222]: audit 2024-03-07T03:39:57.338096+0000 mon.a (mon.0) 1626 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:58.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:57 smithi195 bash[66542]: audit 2024-03-07T03:39:57.338096+0000 mon.a (mon.0) 1626 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:39:59.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:39:58 smithi148 bash[79819]: cluster 2024-03-07T03:39:57.024720+0000 mgr.y (mgr.34101) 780 : cluster 0 pgmap v1330: 105 pgs: 105 active+clean; 124 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 3.0 MiB/s wr, 41 op/s 2024-03-07T03:39:59.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:39:58 smithi148 bash[82222]: cluster 2024-03-07T03:39:57.024720+0000 mgr.y (mgr.34101) 780 : cluster 0 pgmap v1330: 105 pgs: 105 active+clean; 124 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 3.0 MiB/s wr, 41 op/s 2024-03-07T03:39:59.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:39:58 smithi195 bash[66542]: cluster 2024-03-07T03:39:57.024720+0000 mgr.y (mgr.34101) 780 : cluster 0 pgmap v1330: 105 pgs: 105 active+clean; 124 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 3.0 MiB/s wr, 41 op/s 2024-03-07T03:40:00.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:00 smithi148 bash[79819]: debug 2024-03-07T03:39:59.999+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:01.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:00 smithi148 bash[79819]: cluster 2024-03-07T03:39:59.025821+0000 mgr.y (mgr.34101) 781 : cluster 0 pgmap v1331: 105 pgs: 105 active+clean; 141 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:40:01.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:00 smithi148 bash[79819]: cluster 2024-03-07T03:40:00.000180+0000 mon.a (mon.0) 1627 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:01.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:00 smithi148 bash[82222]: cluster 2024-03-07T03:39:59.025821+0000 mgr.y (mgr.34101) 781 : cluster 0 pgmap v1331: 105 pgs: 105 active+clean; 141 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:40:01.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:00 smithi148 bash[82222]: cluster 2024-03-07T03:40:00.000180+0000 mon.a (mon.0) 1627 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:01.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:00 smithi195 bash[66542]: cluster 2024-03-07T03:39:59.025821+0000 mgr.y (mgr.34101) 781 : cluster 0 pgmap v1331: 105 pgs: 105 active+clean; 141 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:40:01.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:00 smithi195 bash[66542]: cluster 2024-03-07T03:40:00.000180+0000 mon.a (mon.0) 1627 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:40:02.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:01 smithi148 bash[79819]: cluster 2024-03-07T03:40:00.770429+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2024-03-07T03:40:02.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:01 smithi148 bash[82222]: cluster 2024-03-07T03:40:00.770429+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2024-03-07T03:40:02.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:01 smithi195 bash[66542]: cluster 2024-03-07T03:40:00.770429+0000 mon.a (mon.0) 1628 : cluster 0 osdmap e932: 8 total, 8 up, 8 in 2024-03-07T03:40:02.563 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:40:02 smithi195 bash[59134]: ts=2024-03-07T03:40:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:03.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:02 smithi148 bash[79819]: cluster 2024-03-07T03:40:01.026490+0000 mgr.y (mgr.34101) 782 : cluster 0 pgmap v1333: 105 pgs: 105 active+clean; 142 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 27 op/s 2024-03-07T03:40:03.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:02 smithi148 bash[82222]: cluster 2024-03-07T03:40:01.026490+0000 mgr.y (mgr.34101) 782 : cluster 0 pgmap v1333: 105 pgs: 105 active+clean; 142 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 27 op/s 2024-03-07T03:40:03.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:02 smithi195 bash[66542]: cluster 2024-03-07T03:40:01.026490+0000 mgr.y (mgr.34101) 782 : cluster 0 pgmap v1333: 105 pgs: 105 active+clean; 142 MiB data, 1017 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 27 op/s 2024-03-07T03:40:05.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:04 smithi148 bash[79819]: cluster 2024-03-07T03:40:03.027124+0000 mgr.y (mgr.34101) 783 : cluster 0 pgmap v1334: 105 pgs: 105 active+clean; 142 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 27 op/s 2024-03-07T03:40:05.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:04 smithi148 bash[82222]: cluster 2024-03-07T03:40:03.027124+0000 mgr.y (mgr.34101) 783 : cluster 0 pgmap v1334: 105 pgs: 105 active+clean; 142 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 27 op/s 2024-03-07T03:40:05.063 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:04 smithi195 bash[66542]: cluster 2024-03-07T03:40:03.027124+0000 mgr.y (mgr.34101) 783 : cluster 0 pgmap v1334: 105 pgs: 105 active+clean; 142 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 27 op/s 2024-03-07T03:40:07.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:06 smithi148 bash[79819]: cluster 2024-03-07T03:40:05.027932+0000 mgr.y (mgr.34101) 784 : cluster 0 pgmap v1335: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 54 op/s 2024-03-07T03:40:07.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:06 smithi148 bash[82222]: cluster 2024-03-07T03:40:05.027932+0000 mgr.y (mgr.34101) 784 : cluster 0 pgmap v1335: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 54 op/s 2024-03-07T03:40:07.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:06 smithi195 bash[66542]: cluster 2024-03-07T03:40:05.027932+0000 mgr.y (mgr.34101) 784 : cluster 0 pgmap v1335: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 54 op/s 2024-03-07T03:40:07.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:40:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:40:07] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.43.0" 2024-03-07T03:40:09.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:08 smithi148 bash[79819]: cluster 2024-03-07T03:40:07.028568+0000 mgr.y (mgr.34101) 785 : cluster 0 pgmap v1336: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.2 MiB/s wr, 45 op/s 2024-03-07T03:40:09.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:08 smithi148 bash[82222]: cluster 2024-03-07T03:40:07.028568+0000 mgr.y (mgr.34101) 785 : cluster 0 pgmap v1336: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.2 MiB/s wr, 45 op/s 2024-03-07T03:40:09.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:08 smithi195 bash[66542]: cluster 2024-03-07T03:40:07.028568+0000 mgr.y (mgr.34101) 785 : cluster 0 pgmap v1336: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.3 MiB/s rd, 6.2 MiB/s wr, 45 op/s 2024-03-07T03:40:10.774 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:40:10 smithi195 bash[49044]: logger=cleanup t=2024-03-07T03:40:10.455911683Z level=info msg="Completed cleanup jobs" duration=342.977072ms 2024-03-07T03:40:11.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:10 smithi148 bash[79819]: cluster 2024-03-07T03:40:09.029309+0000 mgr.y (mgr.34101) 786 : cluster 0 pgmap v1337: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.7 MiB/s wr, 39 op/s 2024-03-07T03:40:11.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:10 smithi148 bash[79819]: audit 2024-03-07T03:40:10.427608+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:40:11.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:10 smithi148 bash[82222]: cluster 2024-03-07T03:40:09.029309+0000 mgr.y (mgr.34101) 786 : cluster 0 pgmap v1337: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.7 MiB/s wr, 39 op/s 2024-03-07T03:40:11.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:10 smithi148 bash[82222]: audit 2024-03-07T03:40:10.427608+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:40:11.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:10 smithi195 bash[66542]: cluster 2024-03-07T03:40:09.029309+0000 mgr.y (mgr.34101) 786 : cluster 0 pgmap v1337: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 439 KiB/s rd, 3.7 MiB/s wr, 39 op/s 2024-03-07T03:40:11.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:10 smithi195 bash[66542]: audit 2024-03-07T03:40:10.427608+0000 mon.a (mon.0) 1629 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:40:12.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:40:12 smithi195 bash[59134]: ts=2024-03-07T03:40:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:12 smithi148 bash[79819]: cluster 2024-03-07T03:40:11.030094+0000 mgr.y (mgr.34101) 787 : cluster 0 pgmap v1338: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 3.5 MiB/s wr, 40 op/s 2024-03-07T03:40:13.040 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:12 smithi148 bash[79819]: audit 2024-03-07T03:40:12.338349+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:12 smithi148 bash[82222]: cluster 2024-03-07T03:40:11.030094+0000 mgr.y (mgr.34101) 787 : cluster 0 pgmap v1338: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 3.5 MiB/s wr, 40 op/s 2024-03-07T03:40:13.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:12 smithi148 bash[82222]: audit 2024-03-07T03:40:12.338349+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:13.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:12 smithi195 bash[66542]: cluster 2024-03-07T03:40:11.030094+0000 mgr.y (mgr.34101) 787 : cluster 0 pgmap v1338: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 3.5 MiB/s wr, 40 op/s 2024-03-07T03:40:13.064 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:12 smithi195 bash[66542]: audit 2024-03-07T03:40:12.338349+0000 mon.a (mon.0) 1630 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:14.290 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:13 smithi148 bash[79819]: cluster 2024-03-07T03:40:13.030684+0000 mgr.y (mgr.34101) 788 : cluster 0 pgmap v1339: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 MiB/s wr, 34 op/s 2024-03-07T03:40:14.290 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:13 smithi148 bash[82222]: cluster 2024-03-07T03:40:13.030684+0000 mgr.y (mgr.34101) 788 : cluster 0 pgmap v1339: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 MiB/s wr, 34 op/s 2024-03-07T03:40:14.314 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:13 smithi195 bash[66542]: cluster 2024-03-07T03:40:13.030684+0000 mgr.y (mgr.34101) 788 : cluster 0 pgmap v1339: 105 pgs: 105 active+clean; 178 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.0 MiB/s wr, 34 op/s 2024-03-07T03:40:14.803 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/4.Shrink (18826 ms) 2024-03-07T03:40:14.803 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/4.ShrinkToOne 2024-03-07T03:40:14.803 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:40:16.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:16 smithi148 bash[79819]: cluster 2024-03-07T03:40:15.031755+0000 mgr.y (mgr.34101) 789 : cluster 0 pgmap v1340: 105 pgs: 105 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.0 MiB/s wr, 48 op/s 2024-03-07T03:40:16.540 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:16 smithi148 bash[82222]: cluster 2024-03-07T03:40:15.031755+0000 mgr.y (mgr.34101) 789 : cluster 0 pgmap v1340: 105 pgs: 105 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.0 MiB/s wr, 48 op/s 2024-03-07T03:40:16.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:16 smithi195 bash[66542]: cluster 2024-03-07T03:40:15.031755+0000 mgr.y (mgr.34101) 789 : cluster 0 pgmap v1340: 105 pgs: 105 active+clean; 210 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 7.0 MiB/s wr, 48 op/s 2024-03-07T03:40:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[79819]: audit 2024-03-07T03:40:16.165368+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[79819]: audit 2024-03-07T03:40:16.173854+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[79819]: audit 2024-03-07T03:40:16.472366+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[79819]: audit 2024-03-07T03:40:16.481504+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[79819]: audit 2024-03-07T03:40:17.116732+0000 mon.a (mon.0) 1635 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:40:17.540 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[79819]: audit 2024-03-07T03:40:17.118491+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[79819]: audit 2024-03-07T03:40:17.127838+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:40:17] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.43.0" 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[82222]: audit 2024-03-07T03:40:16.165368+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[82222]: audit 2024-03-07T03:40:16.173854+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[82222]: audit 2024-03-07T03:40:16.472366+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[82222]: audit 2024-03-07T03:40:16.481504+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[82222]: audit 2024-03-07T03:40:17.116732+0000 mon.a (mon.0) 1635 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[82222]: audit 2024-03-07T03:40:17.118491+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:40:17.541 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:17 smithi148 bash[82222]: audit 2024-03-07T03:40:17.127838+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:17 smithi195 bash[66542]: audit 2024-03-07T03:40:16.165368+0000 mon.a (mon.0) 1631 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:17 smithi195 bash[66542]: audit 2024-03-07T03:40:16.173854+0000 mon.a (mon.0) 1632 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:17 smithi195 bash[66542]: audit 2024-03-07T03:40:16.472366+0000 mon.a (mon.0) 1633 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:17 smithi195 bash[66542]: audit 2024-03-07T03:40:16.481504+0000 mon.a (mon.0) 1634 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:17 smithi195 bash[66542]: audit 2024-03-07T03:40:17.116732+0000 mon.a (mon.0) 1635 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:40:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:17 smithi195 bash[66542]: audit 2024-03-07T03:40:17.118491+0000 mon.a (mon.0) 1636 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:40:17.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:17 smithi195 bash[66542]: audit 2024-03-07T03:40:17.127838+0000 mon.a (mon.0) 1637 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:40:18.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:18 smithi148 bash[79819]: cluster 2024-03-07T03:40:17.032664+0000 mgr.y (mgr.34101) 790 : cluster 0 pgmap v1341: 105 pgs: 105 active+clean; 211 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:40:18.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:18 smithi148 bash[82222]: cluster 2024-03-07T03:40:17.032664+0000 mgr.y (mgr.34101) 790 : cluster 0 pgmap v1341: 105 pgs: 105 active+clean; 211 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:40:18.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:18 smithi195 bash[66542]: cluster 2024-03-07T03:40:17.032664+0000 mgr.y (mgr.34101) 790 : cluster 0 pgmap v1341: 105 pgs: 105 active+clean; 211 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:40:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:20 smithi148 bash[79819]: cluster 2024-03-07T03:40:19.033752+0000 mgr.y (mgr.34101) 791 : cluster 0 pgmap v1342: 105 pgs: 105 active+clean; 223 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.8 MiB/s wr, 33 op/s 2024-03-07T03:40:20.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:20 smithi148 bash[79819]: cluster 2024-03-07T03:40:19.496039+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2024-03-07T03:40:20.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:20 smithi148 bash[82222]: cluster 2024-03-07T03:40:19.033752+0000 mgr.y (mgr.34101) 791 : cluster 0 pgmap v1342: 105 pgs: 105 active+clean; 223 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.8 MiB/s wr, 33 op/s 2024-03-07T03:40:20.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:20 smithi148 bash[82222]: cluster 2024-03-07T03:40:19.496039+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2024-03-07T03:40:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:20 smithi195 bash[66542]: cluster 2024-03-07T03:40:19.033752+0000 mgr.y (mgr.34101) 791 : cluster 0 pgmap v1342: 105 pgs: 105 active+clean; 223 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.8 MiB/s wr, 33 op/s 2024-03-07T03:40:20.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:20 smithi195 bash[66542]: cluster 2024-03-07T03:40:19.496039+0000 mon.a (mon.0) 1638 : cluster 0 osdmap e933: 8 total, 8 up, 8 in 2024-03-07T03:40:22.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:22 smithi195 bash[66542]: cluster 2024-03-07T03:40:21.034483+0000 mgr.y (mgr.34101) 792 : cluster 0 pgmap v1344: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 34 op/s 2024-03-07T03:40:22.564 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:40:22 smithi195 bash[59134]: ts=2024-03-07T03:40:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:22.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:22 smithi148 bash[79819]: cluster 2024-03-07T03:40:21.034483+0000 mgr.y (mgr.34101) 792 : cluster 0 pgmap v1344: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 34 op/s 2024-03-07T03:40:22.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:22 smithi148 bash[82222]: cluster 2024-03-07T03:40:21.034483+0000 mgr.y (mgr.34101) 792 : cluster 0 pgmap v1344: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 34 op/s 2024-03-07T03:40:24.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:24 smithi148 bash[79819]: cluster 2024-03-07T03:40:23.035085+0000 mgr.y (mgr.34101) 793 : cluster 0 pgmap v1345: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 34 op/s 2024-03-07T03:40:24.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:24 smithi148 bash[82222]: cluster 2024-03-07T03:40:23.035085+0000 mgr.y (mgr.34101) 793 : cluster 0 pgmap v1345: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 34 op/s 2024-03-07T03:40:24.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:24 smithi195 bash[66542]: cluster 2024-03-07T03:40:23.035085+0000 mgr.y (mgr.34101) 793 : cluster 0 pgmap v1345: 105 pgs: 105 active+clean; 235 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 34 op/s 2024-03-07T03:40:26.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:26 smithi148 bash[79819]: cluster 2024-03-07T03:40:25.035825+0000 mgr.y (mgr.34101) 794 : cluster 0 pgmap v1346: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 48 op/s 2024-03-07T03:40:26.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:26 smithi148 bash[82222]: cluster 2024-03-07T03:40:25.035825+0000 mgr.y (mgr.34101) 794 : cluster 0 pgmap v1346: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 48 op/s 2024-03-07T03:40:26.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:26 smithi195 bash[66542]: cluster 2024-03-07T03:40:25.035825+0000 mgr.y (mgr.34101) 794 : cluster 0 pgmap v1346: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 48 op/s 2024-03-07T03:40:27.496 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:40:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:40:27] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.43.0" 2024-03-07T03:40:27.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:27 smithi148 bash[79819]: audit 2024-03-07T03:40:27.338243+0000 mon.a (mon.0) 1639 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:27.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:27 smithi148 bash[82222]: audit 2024-03-07T03:40:27.338243+0000 mon.a (mon.0) 1639 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:27.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:27 smithi195 bash[66542]: audit 2024-03-07T03:40:27.338243+0000 mon.a (mon.0) 1639 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:28.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:28 smithi148 bash[79819]: cluster 2024-03-07T03:40:27.036466+0000 mgr.y (mgr.34101) 795 : cluster 0 pgmap v1347: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:40:28.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:28 smithi148 bash[82222]: cluster 2024-03-07T03:40:27.036466+0000 mgr.y (mgr.34101) 795 : cluster 0 pgmap v1347: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:40:28.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:28 smithi195 bash[66542]: cluster 2024-03-07T03:40:27.036466+0000 mgr.y (mgr.34101) 795 : cluster 0 pgmap v1347: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:40:30.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:30 smithi148 bash[79819]: cluster 2024-03-07T03:40:29.037224+0000 mgr.y (mgr.34101) 796 : cluster 0 pgmap v1348: 105 pgs: 105 active+clean; 244 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 32 op/s 2024-03-07T03:40:30.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:30 smithi148 bash[82222]: cluster 2024-03-07T03:40:29.037224+0000 mgr.y (mgr.34101) 796 : cluster 0 pgmap v1348: 105 pgs: 105 active+clean; 244 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 32 op/s 2024-03-07T03:40:30.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:30 smithi195 bash[66542]: cluster 2024-03-07T03:40:29.037224+0000 mgr.y (mgr.34101) 796 : cluster 0 pgmap v1348: 105 pgs: 105 active+clean; 244 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 32 op/s 2024-03-07T03:40:32.564 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:32 smithi195 bash[66542]: cluster 2024-03-07T03:40:31.038021+0000 mgr.y (mgr.34101) 797 : cluster 0 pgmap v1349: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.2 MiB/s wr, 28 op/s 2024-03-07T03:40:32.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:40:32 smithi195 bash[59134]: ts=2024-03-07T03:40:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:32.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:32 smithi148 bash[79819]: cluster 2024-03-07T03:40:31.038021+0000 mgr.y (mgr.34101) 797 : cluster 0 pgmap v1349: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.2 MiB/s wr, 28 op/s 2024-03-07T03:40:32.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:32 smithi148 bash[82222]: cluster 2024-03-07T03:40:31.038021+0000 mgr.y (mgr.34101) 797 : cluster 0 pgmap v1349: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.2 MiB/s wr, 28 op/s 2024-03-07T03:40:34.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:34 smithi148 bash[79819]: cluster 2024-03-07T03:40:33.038783+0000 mgr.y (mgr.34101) 798 : cluster 0 pgmap v1350: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.1 MiB/s wr, 26 op/s 2024-03-07T03:40:34.790 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:34 smithi148 bash[82222]: cluster 2024-03-07T03:40:33.038783+0000 mgr.y (mgr.34101) 798 : cluster 0 pgmap v1350: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.1 MiB/s wr, 26 op/s 2024-03-07T03:40:34.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:34 smithi195 bash[66542]: cluster 2024-03-07T03:40:33.038783+0000 mgr.y (mgr.34101) 798 : cluster 0 pgmap v1350: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.1 MiB/s wr, 26 op/s 2024-03-07T03:40:35.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/4.ShrinkToOne (20485 ms) 2024-03-07T03:40:35.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/4.ShrinkToOneAfterSnapshot 2024-03-07T03:40:35.289 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:40:36.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:36 smithi148 bash[79819]: cluster 2024-03-07T03:40:35.039591+0000 mgr.y (mgr.34101) 799 : cluster 0 pgmap v1351: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-03-07T03:40:36.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:36 smithi148 bash[82222]: cluster 2024-03-07T03:40:35.039591+0000 mgr.y (mgr.34101) 799 : cluster 0 pgmap v1351: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-03-07T03:40:36.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:36 smithi195 bash[66542]: cluster 2024-03-07T03:40:35.039591+0000 mgr.y (mgr.34101) 799 : cluster 0 pgmap v1351: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.4 MiB/s wr, 44 op/s 2024-03-07T03:40:37.540 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:40:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:40:37] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.43.0" 2024-03-07T03:40:38.790 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:38 smithi148 bash[79819]: cluster 2024-03-07T03:40:37.040485+0000 mgr.y (mgr.34101) 800 : cluster 0 pgmap v1352: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 19 op/s 2024-03-07T03:40:38.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:38 smithi148 bash[82222]: cluster 2024-03-07T03:40:37.040485+0000 mgr.y (mgr.34101) 800 : cluster 0 pgmap v1352: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 19 op/s 2024-03-07T03:40:38.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:38 smithi195 bash[66542]: cluster 2024-03-07T03:40:37.040485+0000 mgr.y (mgr.34101) 800 : cluster 0 pgmap v1352: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 19 op/s 2024-03-07T03:40:40.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:40 smithi148 bash[79819]: cluster 2024-03-07T03:40:39.041633+0000 mgr.y (mgr.34101) 801 : cluster 0 pgmap v1353: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:40:40.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:40 smithi148 bash[79819]: cluster 2024-03-07T03:40:39.543983+0000 mon.a (mon.0) 1640 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2024-03-07T03:40:40.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:40 smithi148 bash[82222]: cluster 2024-03-07T03:40:39.041633+0000 mgr.y (mgr.34101) 801 : cluster 0 pgmap v1353: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:40:40.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:40 smithi148 bash[82222]: cluster 2024-03-07T03:40:39.543983+0000 mon.a (mon.0) 1640 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2024-03-07T03:40:40.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:40 smithi195 bash[66542]: cluster 2024-03-07T03:40:39.041633+0000 mgr.y (mgr.34101) 801 : cluster 0 pgmap v1353: 105 pgs: 105 active+clean; 260 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:40:40.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:40 smithi195 bash[66542]: cluster 2024-03-07T03:40:39.543983+0000 mon.a (mon.0) 1640 : cluster 0 osdmap e934: 8 total, 8 up, 8 in 2024-03-07T03:40:42.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:42 smithi195 bash[66542]: cluster 2024-03-07T03:40:41.042374+0000 mgr.y (mgr.34101) 802 : cluster 0 pgmap v1355: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 28 op/s 2024-03-07T03:40:42.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:42 smithi195 bash[66542]: audit 2024-03-07T03:40:42.338263+0000 mon.a (mon.0) 1641 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:42.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:40:42 smithi195 bash[59134]: ts=2024-03-07T03:40:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:42.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:42 smithi148 bash[79819]: cluster 2024-03-07T03:40:41.042374+0000 mgr.y (mgr.34101) 802 : cluster 0 pgmap v1355: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 28 op/s 2024-03-07T03:40:42.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:42 smithi148 bash[79819]: audit 2024-03-07T03:40:42.338263+0000 mon.a (mon.0) 1641 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:42 smithi148 bash[82222]: cluster 2024-03-07T03:40:41.042374+0000 mgr.y (mgr.34101) 802 : cluster 0 pgmap v1355: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 28 op/s 2024-03-07T03:40:42.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:42 smithi148 bash[82222]: audit 2024-03-07T03:40:42.338263+0000 mon.a (mon.0) 1641 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:44.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:44 smithi148 bash[79819]: cluster 2024-03-07T03:40:43.042962+0000 mgr.y (mgr.34101) 803 : cluster 0 pgmap v1356: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 29 op/s 2024-03-07T03:40:44.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:44 smithi148 bash[82222]: cluster 2024-03-07T03:40:43.042962+0000 mgr.y (mgr.34101) 803 : cluster 0 pgmap v1356: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 29 op/s 2024-03-07T03:40:44.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:44 smithi195 bash[66542]: cluster 2024-03-07T03:40:43.042962+0000 mgr.y (mgr.34101) 803 : cluster 0 pgmap v1356: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 29 op/s 2024-03-07T03:40:46.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:46 smithi148 bash[79819]: cluster 2024-03-07T03:40:45.043743+0000 mgr.y (mgr.34101) 804 : cluster 0 pgmap v1357: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 3.3 MiB/s wr, 40 op/s 2024-03-07T03:40:46.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:46 smithi148 bash[82222]: cluster 2024-03-07T03:40:45.043743+0000 mgr.y (mgr.34101) 804 : cluster 0 pgmap v1357: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 3.3 MiB/s wr, 40 op/s 2024-03-07T03:40:46.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:46 smithi195 bash[66542]: cluster 2024-03-07T03:40:45.043743+0000 mgr.y (mgr.34101) 804 : cluster 0 pgmap v1357: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 468 KiB/s rd, 3.3 MiB/s wr, 40 op/s 2024-03-07T03:40:47.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:40:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:40:47] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:40:48.791 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:48 smithi148 bash[79819]: cluster 2024-03-07T03:40:47.044404+0000 mgr.y (mgr.34101) 805 : cluster 0 pgmap v1358: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:40:48.791 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:48 smithi148 bash[82222]: cluster 2024-03-07T03:40:47.044404+0000 mgr.y (mgr.34101) 805 : cluster 0 pgmap v1358: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:40:48.814 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:48 smithi195 bash[66542]: cluster 2024-03-07T03:40:47.044404+0000 mgr.y (mgr.34101) 805 : cluster 0 pgmap v1358: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.9 MiB/s wr, 40 op/s 2024-03-07T03:40:50.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:50 smithi195 bash[66542]: cluster 2024-03-07T03:40:49.045171+0000 mgr.y (mgr.34101) 806 : cluster 0 pgmap v1359: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 34 op/s 2024-03-07T03:40:51.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:50 smithi148 bash[79819]: cluster 2024-03-07T03:40:49.045171+0000 mgr.y (mgr.34101) 806 : cluster 0 pgmap v1359: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 34 op/s 2024-03-07T03:40:51.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:50 smithi148 bash[82222]: cluster 2024-03-07T03:40:49.045171+0000 mgr.y (mgr.34101) 806 : cluster 0 pgmap v1359: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 34 op/s 2024-03-07T03:40:52.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:52 smithi195 bash[66542]: cluster 2024-03-07T03:40:51.045762+0000 mgr.y (mgr.34101) 807 : cluster 0 pgmap v1360: 105 pgs: 105 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.9 MiB/s wr, 30 op/s 2024-03-07T03:40:52.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:40:52 smithi195 bash[59134]: ts=2024-03-07T03:40:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:40:53.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:52 smithi148 bash[82222]: cluster 2024-03-07T03:40:51.045762+0000 mgr.y (mgr.34101) 807 : cluster 0 pgmap v1360: 105 pgs: 105 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.9 MiB/s wr, 30 op/s 2024-03-07T03:40:53.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:52 smithi148 bash[79819]: cluster 2024-03-07T03:40:51.045762+0000 mgr.y (mgr.34101) 807 : cluster 0 pgmap v1360: 105 pgs: 105 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.9 MiB/s wr, 30 op/s 2024-03-07T03:40:54.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:54 smithi195 bash[66542]: cluster 2024-03-07T03:40:53.046244+0000 mgr.y (mgr.34101) 808 : cluster 0 pgmap v1361: 105 pgs: 105 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 31 op/s 2024-03-07T03:40:55.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:54 smithi148 bash[79819]: cluster 2024-03-07T03:40:53.046244+0000 mgr.y (mgr.34101) 808 : cluster 0 pgmap v1361: 105 pgs: 105 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 31 op/s 2024-03-07T03:40:55.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:54 smithi148 bash[82222]: cluster 2024-03-07T03:40:53.046244+0000 mgr.y (mgr.34101) 808 : cluster 0 pgmap v1361: 105 pgs: 105 active+clean; 293 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 31 op/s 2024-03-07T03:40:56.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:55 smithi148 bash[79819]: cluster 2024-03-07T03:40:54.577334+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2024-03-07T03:40:56.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:55 smithi148 bash[82222]: cluster 2024-03-07T03:40:54.577334+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2024-03-07T03:40:56.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:55 smithi195 bash[66542]: cluster 2024-03-07T03:40:54.577334+0000 mon.a (mon.0) 1642 : cluster 0 osdmap e935: 8 total, 8 up, 8 in 2024-03-07T03:40:56.970 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/4.ShrinkToOneAfterSnapshot (21681 ms) 2024-03-07T03:40:56.970 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/4.MinOverlap 2024-03-07T03:40:56.970 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:40:57.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:56 smithi148 bash[79819]: cluster 2024-03-07T03:40:55.046690+0000 mgr.y (mgr.34101) 809 : cluster 0 pgmap v1363: 105 pgs: 105 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:40:57.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:56 smithi148 bash[82222]: cluster 2024-03-07T03:40:55.046690+0000 mgr.y (mgr.34101) 809 : cluster 0 pgmap v1363: 105 pgs: 105 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:40:57.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:56 smithi195 bash[66542]: cluster 2024-03-07T03:40:55.046690+0000 mgr.y (mgr.34101) 809 : cluster 0 pgmap v1363: 105 pgs: 105 active+clean; 305 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:40:57.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:40:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:40:57] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.43.0" 2024-03-07T03:40:58.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:57 smithi148 bash[79819]: audit 2024-03-07T03:40:57.338587+0000 mon.a (mon.0) 1643 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:58.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:57 smithi148 bash[82222]: audit 2024-03-07T03:40:57.338587+0000 mon.a (mon.0) 1643 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:58.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:57 smithi195 bash[66542]: audit 2024-03-07T03:40:57.338587+0000 mon.a (mon.0) 1643 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:40:59.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:40:58 smithi148 bash[79819]: cluster 2024-03-07T03:40:57.047554+0000 mgr.y (mgr.34101) 810 : cluster 0 pgmap v1364: 105 pgs: 105 active+clean; 330 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 994 KiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-03-07T03:40:59.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:40:58 smithi148 bash[82222]: cluster 2024-03-07T03:40:57.047554+0000 mgr.y (mgr.34101) 810 : cluster 0 pgmap v1364: 105 pgs: 105 active+clean; 330 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 994 KiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-03-07T03:40:59.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:40:58 smithi195 bash[66542]: cluster 2024-03-07T03:40:57.047554+0000 mgr.y (mgr.34101) 810 : cluster 0 pgmap v1364: 105 pgs: 105 active+clean; 330 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 994 KiB/s rd, 4.1 MiB/s wr, 23 op/s 2024-03-07T03:41:01.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:00 smithi148 bash[79819]: cluster 2024-03-07T03:40:59.048665+0000 mgr.y (mgr.34101) 811 : cluster 0 pgmap v1365: 105 pgs: 105 active+clean; 374 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 45 op/s 2024-03-07T03:41:01.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:00 smithi148 bash[82222]: cluster 2024-03-07T03:40:59.048665+0000 mgr.y (mgr.34101) 811 : cluster 0 pgmap v1365: 105 pgs: 105 active+clean; 374 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 45 op/s 2024-03-07T03:41:01.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:00 smithi195 bash[66542]: cluster 2024-03-07T03:40:59.048665+0000 mgr.y (mgr.34101) 811 : cluster 0 pgmap v1365: 105 pgs: 105 active+clean; 374 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.5 MiB/s wr, 45 op/s 2024-03-07T03:41:02.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:01 smithi148 bash[79819]: cluster 2024-03-07T03:41:00.600976+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2024-03-07T03:41:02.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:01 smithi148 bash[82222]: cluster 2024-03-07T03:41:00.600976+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2024-03-07T03:41:02.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:01 smithi195 bash[66542]: cluster 2024-03-07T03:41:00.600976+0000 mon.a (mon.0) 1644 : cluster 0 osdmap e936: 8 total, 8 up, 8 in 2024-03-07T03:41:02.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:41:02 smithi195 bash[59134]: ts=2024-03-07T03:41:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:03.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:02 smithi148 bash[79819]: cluster 2024-03-07T03:41:01.049316+0000 mgr.y (mgr.34101) 812 : cluster 0 pgmap v1367: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 10 MiB/s wr, 56 op/s 2024-03-07T03:41:03.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:02 smithi148 bash[82222]: cluster 2024-03-07T03:41:01.049316+0000 mgr.y (mgr.34101) 812 : cluster 0 pgmap v1367: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 10 MiB/s wr, 56 op/s 2024-03-07T03:41:03.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:02 smithi195 bash[66542]: cluster 2024-03-07T03:41:01.049316+0000 mgr.y (mgr.34101) 812 : cluster 0 pgmap v1367: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 10 MiB/s wr, 56 op/s 2024-03-07T03:41:05.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:04 smithi148 bash[79819]: cluster 2024-03-07T03:41:03.049985+0000 mgr.y (mgr.34101) 813 : cluster 0 pgmap v1368: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 8.1 MiB/s wr, 54 op/s 2024-03-07T03:41:05.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:04 smithi148 bash[82222]: cluster 2024-03-07T03:41:03.049985+0000 mgr.y (mgr.34101) 813 : cluster 0 pgmap v1368: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 8.1 MiB/s wr, 54 op/s 2024-03-07T03:41:05.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:04 smithi195 bash[66542]: cluster 2024-03-07T03:41:03.049985+0000 mgr.y (mgr.34101) 813 : cluster 0 pgmap v1368: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 8.1 MiB/s wr, 54 op/s 2024-03-07T03:41:07.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:06 smithi148 bash[79819]: cluster 2024-03-07T03:41:05.050793+0000 mgr.y (mgr.34101) 814 : cluster 0 pgmap v1369: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.6 MiB/s wr, 49 op/s 2024-03-07T03:41:07.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:06 smithi148 bash[82222]: cluster 2024-03-07T03:41:05.050793+0000 mgr.y (mgr.34101) 814 : cluster 0 pgmap v1369: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.6 MiB/s wr, 49 op/s 2024-03-07T03:41:07.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:06 smithi195 bash[66542]: cluster 2024-03-07T03:41:05.050793+0000 mgr.y (mgr.34101) 814 : cluster 0 pgmap v1369: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.6 MiB/s wr, 49 op/s 2024-03-07T03:41:07.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:41:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:41:07] "GET /metrics HTTP/1.1" 200 36730 "" "Prometheus/2.43.0" 2024-03-07T03:41:09.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:08 smithi148 bash[79819]: cluster 2024-03-07T03:41:07.051437+0000 mgr.y (mgr.34101) 815 : cluster 0 pgmap v1370: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.3 MiB/s wr, 61 op/s 2024-03-07T03:41:09.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:08 smithi148 bash[82222]: cluster 2024-03-07T03:41:07.051437+0000 mgr.y (mgr.34101) 815 : cluster 0 pgmap v1370: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.3 MiB/s wr, 61 op/s 2024-03-07T03:41:09.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:08 smithi195 bash[66542]: cluster 2024-03-07T03:41:07.051437+0000 mgr.y (mgr.34101) 815 : cluster 0 pgmap v1370: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.3 MiB/s wr, 61 op/s 2024-03-07T03:41:11.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:10 smithi148 bash[79819]: cluster 2024-03-07T03:41:09.052265+0000 mgr.y (mgr.34101) 816 : cluster 0 pgmap v1371: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 39 op/s 2024-03-07T03:41:11.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:10 smithi148 bash[82222]: cluster 2024-03-07T03:41:09.052265+0000 mgr.y (mgr.34101) 816 : cluster 0 pgmap v1371: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 39 op/s 2024-03-07T03:41:11.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:10 smithi195 bash[66542]: cluster 2024-03-07T03:41:09.052265+0000 mgr.y (mgr.34101) 816 : cluster 0 pgmap v1371: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 2.9 MiB/s wr, 39 op/s 2024-03-07T03:41:12.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:41:12 smithi195 bash[59134]: ts=2024-03-07T03:41:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:13.041 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:12 smithi148 bash[82222]: cluster 2024-03-07T03:41:11.052845+0000 mgr.y (mgr.34101) 817 : cluster 0 pgmap v1372: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-03-07T03:41:13.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:12 smithi148 bash[82222]: audit 2024-03-07T03:41:12.339099+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:13.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:12 smithi148 bash[79819]: cluster 2024-03-07T03:41:11.052845+0000 mgr.y (mgr.34101) 817 : cluster 0 pgmap v1372: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-03-07T03:41:13.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:12 smithi148 bash[79819]: audit 2024-03-07T03:41:12.339099+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:13.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:12 smithi195 bash[66542]: cluster 2024-03-07T03:41:11.052845+0000 mgr.y (mgr.34101) 817 : cluster 0 pgmap v1372: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-03-07T03:41:13.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:12 smithi195 bash[66542]: audit 2024-03-07T03:41:12.339099+0000 mon.a (mon.0) 1645 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:15.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:14 smithi148 bash[79819]: cluster 2024-03-07T03:41:13.053515+0000 mgr.y (mgr.34101) 818 : cluster 0 pgmap v1373: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-03-07T03:41:15.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:14 smithi148 bash[82222]: cluster 2024-03-07T03:41:13.053515+0000 mgr.y (mgr.34101) 818 : cluster 0 pgmap v1373: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-03-07T03:41:15.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:14 smithi195 bash[66542]: cluster 2024-03-07T03:41:13.053515+0000 mgr.y (mgr.34101) 818 : cluster 0 pgmap v1373: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 22 KiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-03-07T03:41:17.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:16 smithi148 bash[79819]: cluster 2024-03-07T03:41:15.055517+0000 mgr.y (mgr.34101) 819 : cluster 0 pgmap v1374: 105 pgs: 105 active+clean; 411 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 3.1 MiB/s wr, 27 op/s 2024-03-07T03:41:17.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:16 smithi148 bash[82222]: cluster 2024-03-07T03:41:15.055517+0000 mgr.y (mgr.34101) 819 : cluster 0 pgmap v1374: 105 pgs: 105 active+clean; 411 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 3.1 MiB/s wr, 27 op/s 2024-03-07T03:41:17.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:16 smithi195 bash[66542]: cluster 2024-03-07T03:41:15.055517+0000 mgr.y (mgr.34101) 819 : cluster 0 pgmap v1374: 105 pgs: 105 active+clean; 411 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 3.1 MiB/s wr, 27 op/s 2024-03-07T03:41:17.210 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/4.MinOverlap (20239 ms) 2024-03-07T03:41:17.211 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/4.ZeroOverlap 2024-03-07T03:41:17.211 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:41:17.541 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:41:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:41:17] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.43.0" 2024-03-07T03:41:18.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:17 smithi148 bash[79819]: audit 2024-03-07T03:41:17.207561+0000 mon.a (mon.0) 1646 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:41:18.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:17 smithi148 bash[82222]: audit 2024-03-07T03:41:17.207561+0000 mon.a (mon.0) 1646 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:41:18.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:17 smithi195 bash[66542]: audit 2024-03-07T03:41:17.207561+0000 mon.a (mon.0) 1646 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:41:19.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:18 smithi148 bash[79819]: cluster 2024-03-07T03:41:17.056400+0000 mgr.y (mgr.34101) 820 : cluster 0 pgmap v1375: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-03-07T03:41:19.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:18 smithi148 bash[79819]: audit 2024-03-07T03:41:17.732755+0000 mon.a (mon.0) 1647 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:19.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:18 smithi148 bash[79819]: audit 2024-03-07T03:41:17.739014+0000 mon.a (mon.0) 1648 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:19.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:18 smithi148 bash[82222]: cluster 2024-03-07T03:41:17.056400+0000 mgr.y (mgr.34101) 820 : cluster 0 pgmap v1375: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-03-07T03:41:19.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:18 smithi148 bash[82222]: audit 2024-03-07T03:41:17.732755+0000 mon.a (mon.0) 1647 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:19.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:18 smithi148 bash[82222]: audit 2024-03-07T03:41:17.739014+0000 mon.a (mon.0) 1648 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:19.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:18 smithi195 bash[66542]: cluster 2024-03-07T03:41:17.056400+0000 mgr.y (mgr.34101) 820 : cluster 0 pgmap v1375: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-03-07T03:41:19.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:18 smithi195 bash[66542]: audit 2024-03-07T03:41:17.732755+0000 mon.a (mon.0) 1647 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:19.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:18 smithi195 bash[66542]: audit 2024-03-07T03:41:17.739014+0000 mon.a (mon.0) 1648 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:21.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:20 smithi148 bash[79819]: cluster 2024-03-07T03:41:19.057465+0000 mgr.y (mgr.34101) 821 : cluster 0 pgmap v1376: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:41:21.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:20 smithi148 bash[82222]: cluster 2024-03-07T03:41:19.057465+0000 mgr.y (mgr.34101) 821 : cluster 0 pgmap v1376: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:41:21.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:20 smithi195 bash[66542]: cluster 2024-03-07T03:41:19.057465+0000 mgr.y (mgr.34101) 821 : cluster 0 pgmap v1376: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:41:22.565 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:41:22 smithi195 bash[59134]: ts=2024-03-07T03:41:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:23.041 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:22 smithi148 bash[79819]: cluster 2024-03-07T03:41:21.062071+0000 mgr.y (mgr.34101) 822 : cluster 0 pgmap v1377: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:41:23.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:22 smithi148 bash[79819]: cluster 2024-03-07T03:41:21.750779+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2024-03-07T03:41:23.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:22 smithi148 bash[82222]: cluster 2024-03-07T03:41:21.062071+0000 mgr.y (mgr.34101) 822 : cluster 0 pgmap v1377: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:41:23.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:22 smithi148 bash[82222]: cluster 2024-03-07T03:41:21.750779+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2024-03-07T03:41:23.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:22 smithi195 bash[66542]: cluster 2024-03-07T03:41:21.062071+0000 mgr.y (mgr.34101) 822 : cluster 0 pgmap v1377: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:41:23.065 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:22 smithi195 bash[66542]: cluster 2024-03-07T03:41:21.750779+0000 mon.a (mon.0) 1649 : cluster 0 osdmap e937: 8 total, 8 up, 8 in 2024-03-07T03:41:24.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: cluster 2024-03-07T03:41:23.062787+0000 mgr.y (mgr.34101) 823 : cluster 0 pgmap v1379: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.0 MiB/s wr, 30 op/s 2024-03-07T03:41:24.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: audit 2024-03-07T03:41:23.257772+0000 mon.a (mon.0) 1650 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: audit 2024-03-07T03:41:23.266710+0000 mon.a (mon.0) 1651 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: audit 2024-03-07T03:41:23.527732+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: audit 2024-03-07T03:41:23.534615+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: audit 2024-03-07T03:41:24.161719+0000 mon.a (mon.0) 1654 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:41:24.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: audit 2024-03-07T03:41:24.162446+0000 mon.a (mon.0) 1655 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[79819]: audit 2024-03-07T03:41:24.167339+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: cluster 2024-03-07T03:41:23.062787+0000 mgr.y (mgr.34101) 823 : cluster 0 pgmap v1379: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.0 MiB/s wr, 30 op/s 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: audit 2024-03-07T03:41:23.257772+0000 mon.a (mon.0) 1650 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: audit 2024-03-07T03:41:23.266710+0000 mon.a (mon.0) 1651 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: audit 2024-03-07T03:41:23.527732+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: audit 2024-03-07T03:41:23.534615+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: audit 2024-03-07T03:41:24.161719+0000 mon.a (mon.0) 1654 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: audit 2024-03-07T03:41:24.162446+0000 mon.a (mon.0) 1655 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:41:24.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:24 smithi148 bash[82222]: audit 2024-03-07T03:41:24.167339+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: cluster 2024-03-07T03:41:23.062787+0000 mgr.y (mgr.34101) 823 : cluster 0 pgmap v1379: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.0 MiB/s wr, 30 op/s 2024-03-07T03:41:24.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: audit 2024-03-07T03:41:23.257772+0000 mon.a (mon.0) 1650 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: audit 2024-03-07T03:41:23.266710+0000 mon.a (mon.0) 1651 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: audit 2024-03-07T03:41:23.527732+0000 mon.a (mon.0) 1652 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: audit 2024-03-07T03:41:23.534615+0000 mon.a (mon.0) 1653 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:24.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: audit 2024-03-07T03:41:24.161719+0000 mon.a (mon.0) 1654 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:41:24.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: audit 2024-03-07T03:41:24.162446+0000 mon.a (mon.0) 1655 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:41:24.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:24 smithi195 bash[66542]: audit 2024-03-07T03:41:24.167339+0000 mon.a (mon.0) 1656 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:26.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:26 smithi148 bash[79819]: cluster 2024-03-07T03:41:25.063523+0000 mgr.y (mgr.34101) 824 : cluster 0 pgmap v1380: 105 pgs: 105 active+clean; 451 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.0 MiB/s wr, 52 op/s 2024-03-07T03:41:26.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:26 smithi148 bash[82222]: cluster 2024-03-07T03:41:25.063523+0000 mgr.y (mgr.34101) 824 : cluster 0 pgmap v1380: 105 pgs: 105 active+clean; 451 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.0 MiB/s wr, 52 op/s 2024-03-07T03:41:26.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:26 smithi195 bash[66542]: cluster 2024-03-07T03:41:25.063523+0000 mgr.y (mgr.34101) 824 : cluster 0 pgmap v1380: 105 pgs: 105 active+clean; 451 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.0 MiB/s wr, 52 op/s 2024-03-07T03:41:27.542 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:41:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:41:27] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.43.0" 2024-03-07T03:41:28.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:28 smithi148 bash[79819]: cluster 2024-03-07T03:41:27.064184+0000 mgr.y (mgr.34101) 825 : cluster 0 pgmap v1381: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 49 op/s 2024-03-07T03:41:28.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:28 smithi148 bash[79819]: audit 2024-03-07T03:41:27.339387+0000 mon.a (mon.0) 1657 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:28.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:28 smithi148 bash[82222]: cluster 2024-03-07T03:41:27.064184+0000 mgr.y (mgr.34101) 825 : cluster 0 pgmap v1381: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 49 op/s 2024-03-07T03:41:28.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:28 smithi148 bash[82222]: audit 2024-03-07T03:41:27.339387+0000 mon.a (mon.0) 1657 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:28.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:28 smithi195 bash[66542]: cluster 2024-03-07T03:41:27.064184+0000 mgr.y (mgr.34101) 825 : cluster 0 pgmap v1381: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 49 op/s 2024-03-07T03:41:28.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:28 smithi195 bash[66542]: audit 2024-03-07T03:41:27.339387+0000 mon.a (mon.0) 1657 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:30.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:30 smithi148 bash[79819]: cluster 2024-03-07T03:41:29.065017+0000 mgr.y (mgr.34101) 826 : cluster 0 pgmap v1382: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 40 op/s 2024-03-07T03:41:30.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:30 smithi148 bash[82222]: cluster 2024-03-07T03:41:29.065017+0000 mgr.y (mgr.34101) 826 : cluster 0 pgmap v1382: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 40 op/s 2024-03-07T03:41:30.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:30 smithi195 bash[66542]: cluster 2024-03-07T03:41:29.065017+0000 mgr.y (mgr.34101) 826 : cluster 0 pgmap v1382: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 40 op/s 2024-03-07T03:41:32.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:32 smithi148 bash[79819]: cluster 2024-03-07T03:41:31.065822+0000 mgr.y (mgr.34101) 827 : cluster 0 pgmap v1383: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-03-07T03:41:32.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:32 smithi148 bash[82222]: cluster 2024-03-07T03:41:31.065822+0000 mgr.y (mgr.34101) 827 : cluster 0 pgmap v1383: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-03-07T03:41:32.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:32 smithi195 bash[66542]: cluster 2024-03-07T03:41:31.065822+0000 mgr.y (mgr.34101) 827 : cluster 0 pgmap v1383: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 3.3 MiB/s wr, 35 op/s 2024-03-07T03:41:32.566 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:41:32 smithi195 bash[59134]: ts=2024-03-07T03:41:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:34.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:34 smithi148 bash[79819]: cluster 2024-03-07T03:41:33.066569+0000 mgr.y (mgr.34101) 828 : cluster 0 pgmap v1384: 105 pgs: 105 active+clean; 460 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.3 MiB/s wr, 31 op/s 2024-03-07T03:41:34.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:34 smithi148 bash[82222]: cluster 2024-03-07T03:41:33.066569+0000 mgr.y (mgr.34101) 828 : cluster 0 pgmap v1384: 105 pgs: 105 active+clean; 460 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.3 MiB/s wr, 31 op/s 2024-03-07T03:41:34.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:34 smithi195 bash[66542]: cluster 2024-03-07T03:41:33.066569+0000 mgr.y (mgr.34101) 828 : cluster 0 pgmap v1384: 105 pgs: 105 active+clean; 460 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 3.3 MiB/s wr, 31 op/s 2024-03-07T03:41:34.892 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/4.ZeroOverlap (17681 ms) 2024-03-07T03:41:35.279 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/4 (138408 ms total) 2024-03-07T03:41:35.279 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:41:35.279 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/5, where TypeParam = UnformattedLUKS2 2024-03-07T03:41:35.310 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/5.Simple 2024-03-07T03:41:36.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:36 smithi148 bash[79819]: cluster 2024-03-07T03:41:35.067736+0000 mgr.y (mgr.34101) 829 : cluster 0 pgmap v1385: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 790 KiB/s rd, 4.4 MiB/s wr, 48 op/s 2024-03-07T03:41:36.542 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:36 smithi148 bash[79819]: cluster 2024-03-07T03:41:35.287738+0000 mon.a (mon.0) 1658 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2024-03-07T03:41:36.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:36 smithi148 bash[82222]: cluster 2024-03-07T03:41:35.067736+0000 mgr.y (mgr.34101) 829 : cluster 0 pgmap v1385: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 790 KiB/s rd, 4.4 MiB/s wr, 48 op/s 2024-03-07T03:41:36.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:36 smithi148 bash[82222]: cluster 2024-03-07T03:41:35.287738+0000 mon.a (mon.0) 1658 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2024-03-07T03:41:36.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:36 smithi195 bash[66542]: cluster 2024-03-07T03:41:35.067736+0000 mgr.y (mgr.34101) 829 : cluster 0 pgmap v1385: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 790 KiB/s rd, 4.4 MiB/s wr, 48 op/s 2024-03-07T03:41:36.565 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:36 smithi195 bash[66542]: cluster 2024-03-07T03:41:35.287738+0000 mon.a (mon.0) 1658 : cluster 0 osdmap e938: 8 total, 8 up, 8 in 2024-03-07T03:41:37.316 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:41:37.542 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:41:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:41:37] "GET /metrics HTTP/1.1" 200 36734 "" "Prometheus/2.43.0" 2024-03-07T03:41:37.542 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:37 smithi148 bash[82222]: cluster 2024-03-07T03:41:36.306879+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2024-03-07T03:41:37.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:37 smithi148 bash[82222]: audit 2024-03-07T03:41:36.314472+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.148:0/2870375588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:37.543 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:37 smithi148 bash[82222]: audit 2024-03-07T03:41:36.315280+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:37.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:37 smithi148 bash[79819]: cluster 2024-03-07T03:41:36.306879+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2024-03-07T03:41:37.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:37 smithi148 bash[79819]: audit 2024-03-07T03:41:36.314472+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.148:0/2870375588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:37.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:37 smithi148 bash[79819]: audit 2024-03-07T03:41:36.315280+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:37.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:37 smithi195 bash[66542]: cluster 2024-03-07T03:41:36.306879+0000 mon.a (mon.0) 1659 : cluster 0 osdmap e939: 8 total, 8 up, 8 in 2024-03-07T03:41:37.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:37 smithi195 bash[66542]: audit 2024-03-07T03:41:36.314472+0000 mon.c (mon.2) 82 : audit 1 from='client.? 172.21.15.148:0/2870375588' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:37.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:37 smithi195 bash[66542]: audit 2024-03-07T03:41:36.315280+0000 mon.a (mon.0) 1660 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:41:38.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:38 smithi195 bash[66542]: cluster 2024-03-07T03:41:37.068593+0000 mgr.y (mgr.34101) 830 : cluster 0 pgmap v1388: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:41:38.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:38 smithi195 bash[66542]: cluster 2024-03-07T03:41:37.299183+0000 mon.a (mon.0) 1661 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:41:38.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:38 smithi195 bash[66542]: audit 2024-03-07T03:41:37.309204+0000 mon.a (mon.0) 1662 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:41:38.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:38 smithi195 bash[66542]: cluster 2024-03-07T03:41:37.320527+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2024-03-07T03:41:38.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[79819]: cluster 2024-03-07T03:41:37.068593+0000 mgr.y (mgr.34101) 830 : cluster 0 pgmap v1388: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:41:38.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[79819]: cluster 2024-03-07T03:41:37.299183+0000 mon.a (mon.0) 1661 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:41:38.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[79819]: audit 2024-03-07T03:41:37.309204+0000 mon.a (mon.0) 1662 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:41:38.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[79819]: cluster 2024-03-07T03:41:37.320527+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2024-03-07T03:41:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[82222]: cluster 2024-03-07T03:41:37.068593+0000 mgr.y (mgr.34101) 830 : cluster 0 pgmap v1388: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:41:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[82222]: cluster 2024-03-07T03:41:37.299183+0000 mon.a (mon.0) 1661 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:41:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[82222]: audit 2024-03-07T03:41:37.309204+0000 mon.a (mon.0) 1662 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:41:38.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:38 smithi148 bash[82222]: cluster 2024-03-07T03:41:37.320527+0000 mon.a (mon.0) 1663 : cluster 0 osdmap e940: 8 total, 8 up, 8 in 2024-03-07T03:41:39.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:39 smithi148 bash[82222]: cluster 2024-03-07T03:41:38.319808+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2024-03-07T03:41:39.793 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:39 smithi148 bash[79819]: cluster 2024-03-07T03:41:38.319808+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2024-03-07T03:41:39.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:39 smithi195 bash[66542]: cluster 2024-03-07T03:41:38.319808+0000 mon.a (mon.0) 1664 : cluster 0 osdmap e941: 8 total, 8 up, 8 in 2024-03-07T03:41:40.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:40 smithi148 bash[79819]: cluster 2024-03-07T03:41:39.069607+0000 mgr.y (mgr.34101) 831 : cluster 0 pgmap v1391: 105 pgs: 4 creating+peering, 8 unknown, 93 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:40.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:40 smithi148 bash[79819]: cluster 2024-03-07T03:41:39.324862+0000 mon.a (mon.0) 1665 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:41:40.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:40 smithi148 bash[79819]: cluster 2024-03-07T03:41:39.349456+0000 mon.a (mon.0) 1666 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2024-03-07T03:41:40.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:40 smithi148 bash[82222]: cluster 2024-03-07T03:41:39.069607+0000 mgr.y (mgr.34101) 831 : cluster 0 pgmap v1391: 105 pgs: 4 creating+peering, 8 unknown, 93 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:40.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:40 smithi148 bash[82222]: cluster 2024-03-07T03:41:39.324862+0000 mon.a (mon.0) 1665 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:41:40.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:40 smithi148 bash[82222]: cluster 2024-03-07T03:41:39.349456+0000 mon.a (mon.0) 1666 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2024-03-07T03:41:40.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:40 smithi195 bash[66542]: cluster 2024-03-07T03:41:39.069607+0000 mgr.y (mgr.34101) 831 : cluster 0 pgmap v1391: 105 pgs: 4 creating+peering, 8 unknown, 93 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:40.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:40 smithi195 bash[66542]: cluster 2024-03-07T03:41:39.324862+0000 mon.a (mon.0) 1665 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:41:40.816 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:40 smithi195 bash[66542]: cluster 2024-03-07T03:41:39.349456+0000 mon.a (mon.0) 1666 : cluster 0 osdmap e942: 8 total, 8 up, 8 in 2024-03-07T03:41:42.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:42 smithi195 bash[66542]: cluster 2024-03-07T03:41:41.070282+0000 mgr.y (mgr.34101) 832 : cluster 0 pgmap v1393: 105 pgs: 4 creating+peering, 101 active+clean; 582 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:42.566 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:41:42 smithi195 bash[59134]: ts=2024-03-07T03:41:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:42.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:42 smithi148 bash[79819]: cluster 2024-03-07T03:41:41.070282+0000 mgr.y (mgr.34101) 832 : cluster 0 pgmap v1393: 105 pgs: 4 creating+peering, 101 active+clean; 582 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:42.792 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:42 smithi148 bash[82222]: cluster 2024-03-07T03:41:41.070282+0000 mgr.y (mgr.34101) 832 : cluster 0 pgmap v1393: 105 pgs: 4 creating+peering, 101 active+clean; 582 KiB data, 827 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:41:43.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:43 smithi148 bash[79819]: audit 2024-03-07T03:41:42.340510+0000 mon.a (mon.0) 1667 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:43.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:43 smithi148 bash[82222]: audit 2024-03-07T03:41:42.340510+0000 mon.a (mon.0) 1667 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:43.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:43 smithi195 bash[66542]: audit 2024-03-07T03:41:42.340510+0000 mon.a (mon.0) 1667 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:44.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:44 smithi148 bash[79819]: cluster 2024-03-07T03:41:43.070876+0000 mgr.y (mgr.34101) 833 : cluster 0 pgmap v1394: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 511 B/s wr, 0 op/s 2024-03-07T03:41:44.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:44 smithi148 bash[82222]: cluster 2024-03-07T03:41:43.070876+0000 mgr.y (mgr.34101) 833 : cluster 0 pgmap v1394: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 511 B/s wr, 0 op/s 2024-03-07T03:41:44.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:44 smithi195 bash[66542]: cluster 2024-03-07T03:41:43.070876+0000 mgr.y (mgr.34101) 833 : cluster 0 pgmap v1394: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 511 B/s wr, 0 op/s 2024-03-07T03:41:46.792 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:46 smithi148 bash[79819]: cluster 2024-03-07T03:41:45.071629+0000 mgr.y (mgr.34101) 834 : cluster 0 pgmap v1395: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:41:46.793 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:46 smithi148 bash[82222]: cluster 2024-03-07T03:41:45.071629+0000 mgr.y (mgr.34101) 834 : cluster 0 pgmap v1395: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:41:46.815 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:46 smithi195 bash[66542]: cluster 2024-03-07T03:41:45.071629+0000 mgr.y (mgr.34101) 834 : cluster 0 pgmap v1395: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:41:47.542 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:41:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:41:47] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2024-03-07T03:41:49.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:48 smithi148 bash[79819]: cluster 2024-03-07T03:41:47.072284+0000 mgr.y (mgr.34101) 835 : cluster 0 pgmap v1396: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 935 B/s wr, 6 op/s 2024-03-07T03:41:49.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:48 smithi148 bash[79819]: cluster 2024-03-07T03:41:47.651447+0000 mon.a (mon.0) 1668 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2024-03-07T03:41:49.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:48 smithi148 bash[82222]: cluster 2024-03-07T03:41:47.072284+0000 mgr.y (mgr.34101) 835 : cluster 0 pgmap v1396: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 935 B/s wr, 6 op/s 2024-03-07T03:41:49.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:48 smithi148 bash[82222]: cluster 2024-03-07T03:41:47.651447+0000 mon.a (mon.0) 1668 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2024-03-07T03:41:49.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:48 smithi195 bash[66542]: cluster 2024-03-07T03:41:47.072284+0000 mgr.y (mgr.34101) 835 : cluster 0 pgmap v1396: 105 pgs: 105 active+clean; 582 KiB data, 603 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 935 B/s wr, 6 op/s 2024-03-07T03:41:49.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:48 smithi195 bash[66542]: cluster 2024-03-07T03:41:47.651447+0000 mon.a (mon.0) 1668 : cluster 0 osdmap e943: 8 total, 8 up, 8 in 2024-03-07T03:41:50.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:49 smithi148 bash[79819]: audit 2024-03-07T03:41:48.698624+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:50.042 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:49 smithi148 bash[82222]: audit 2024-03-07T03:41:48.698624+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:50.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:49 smithi195 bash[66542]: audit 2024-03-07T03:41:48.698624+0000 mon.a (mon.0) 1669 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:41:51.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:50 smithi148 bash[79819]: cluster 2024-03-07T03:41:49.073047+0000 mgr.y (mgr.34101) 836 : cluster 0 pgmap v1398: 105 pgs: 105 active+clean; 26 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-03-07T03:41:51.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:50 smithi148 bash[82222]: cluster 2024-03-07T03:41:49.073047+0000 mgr.y (mgr.34101) 836 : cluster 0 pgmap v1398: 105 pgs: 105 active+clean; 26 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-03-07T03:41:51.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:50 smithi195 bash[66542]: cluster 2024-03-07T03:41:49.073047+0000 mgr.y (mgr.34101) 836 : cluster 0 pgmap v1398: 105 pgs: 105 active+clean; 26 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-03-07T03:41:52.566 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:41:52 smithi195 bash[59134]: ts=2024-03-07T03:41:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:41:53.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:52 smithi148 bash[79819]: cluster 2024-03-07T03:41:51.073642+0000 mgr.y (mgr.34101) 837 : cluster 0 pgmap v1399: 105 pgs: 105 active+clean; 34 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:41:53.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:52 smithi148 bash[82222]: cluster 2024-03-07T03:41:51.073642+0000 mgr.y (mgr.34101) 837 : cluster 0 pgmap v1399: 105 pgs: 105 active+clean; 34 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:41:53.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:52 smithi195 bash[66542]: cluster 2024-03-07T03:41:51.073642+0000 mgr.y (mgr.34101) 837 : cluster 0 pgmap v1399: 105 pgs: 105 active+clean; 34 MiB data, 709 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-03-07T03:41:53.627 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/5.Simple (18313 ms) 2024-03-07T03:41:53.627 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/5.Grow 2024-03-07T03:41:53.627 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:41:55.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:54 smithi148 bash[82222]: cluster 2024-03-07T03:41:53.074179+0000 mgr.y (mgr.34101) 838 : cluster 0 pgmap v1400: 105 pgs: 105 active+clean; 42 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-03-07T03:41:55.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:54 smithi148 bash[79819]: cluster 2024-03-07T03:41:53.074179+0000 mgr.y (mgr.34101) 838 : cluster 0 pgmap v1400: 105 pgs: 105 active+clean; 42 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-03-07T03:41:55.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:54 smithi195 bash[66542]: cluster 2024-03-07T03:41:53.074179+0000 mgr.y (mgr.34101) 838 : cluster 0 pgmap v1400: 105 pgs: 105 active+clean; 42 MiB data, 726 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-03-07T03:41:57.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:56 smithi148 bash[79819]: cluster 2024-03-07T03:41:55.074869+0000 mgr.y (mgr.34101) 839 : cluster 0 pgmap v1401: 105 pgs: 105 active+clean; 63 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 6.2 MiB/s wr, 53 op/s 2024-03-07T03:41:57.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:56 smithi148 bash[82222]: cluster 2024-03-07T03:41:55.074869+0000 mgr.y (mgr.34101) 839 : cluster 0 pgmap v1401: 105 pgs: 105 active+clean; 63 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 6.2 MiB/s wr, 53 op/s 2024-03-07T03:41:57.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:56 smithi195 bash[66542]: cluster 2024-03-07T03:41:55.074869+0000 mgr.y (mgr.34101) 839 : cluster 0 pgmap v1401: 105 pgs: 105 active+clean; 63 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 6.2 MiB/s wr, 53 op/s 2024-03-07T03:41:57.542 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:41:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:41:57] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2024-03-07T03:41:58.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:57 smithi148 bash[79819]: audit 2024-03-07T03:41:57.340132+0000 mon.a (mon.0) 1670 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:58.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:57 smithi148 bash[82222]: audit 2024-03-07T03:41:57.340132+0000 mon.a (mon.0) 1670 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:58.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:57 smithi195 bash[66542]: audit 2024-03-07T03:41:57.340132+0000 mon.a (mon.0) 1670 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:41:59.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:41:58 smithi148 bash[79819]: cluster 2024-03-07T03:41:57.075516+0000 mgr.y (mgr.34101) 840 : cluster 0 pgmap v1402: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:41:59.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:41:58 smithi148 bash[82222]: cluster 2024-03-07T03:41:57.075516+0000 mgr.y (mgr.34101) 840 : cluster 0 pgmap v1402: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:41:59.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:41:58 smithi195 bash[66542]: cluster 2024-03-07T03:41:57.075516+0000 mgr.y (mgr.34101) 840 : cluster 0 pgmap v1402: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-03-07T03:42:01.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:00 smithi148 bash[79819]: cluster 2024-03-07T03:41:59.076282+0000 mgr.y (mgr.34101) 841 : cluster 0 pgmap v1403: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 MiB/s wr, 60 op/s 2024-03-07T03:42:01.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:00 smithi148 bash[82222]: cluster 2024-03-07T03:41:59.076282+0000 mgr.y (mgr.34101) 841 : cluster 0 pgmap v1403: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 MiB/s wr, 60 op/s 2024-03-07T03:42:01.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:00 smithi195 bash[66542]: cluster 2024-03-07T03:41:59.076282+0000 mgr.y (mgr.34101) 841 : cluster 0 pgmap v1403: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.2 MiB/s wr, 60 op/s 2024-03-07T03:42:02.566 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:42:02 smithi195 bash[59134]: ts=2024-03-07T03:42:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:03.042 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:02 smithi148 bash[79819]: cluster 2024-03-07T03:42:01.076844+0000 mgr.y (mgr.34101) 842 : cluster 0 pgmap v1404: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-03-07T03:42:03.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:02 smithi148 bash[82222]: cluster 2024-03-07T03:42:01.076844+0000 mgr.y (mgr.34101) 842 : cluster 0 pgmap v1404: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-03-07T03:42:03.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:02 smithi195 bash[66542]: cluster 2024-03-07T03:42:01.076844+0000 mgr.y (mgr.34101) 842 : cluster 0 pgmap v1404: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-03-07T03:42:05.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:04 smithi148 bash[79819]: cluster 2024-03-07T03:42:03.077404+0000 mgr.y (mgr.34101) 843 : cluster 0 pgmap v1405: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-03-07T03:42:05.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:04 smithi148 bash[82222]: cluster 2024-03-07T03:42:03.077404+0000 mgr.y (mgr.34101) 843 : cluster 0 pgmap v1405: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-03-07T03:42:05.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:04 smithi195 bash[66542]: cluster 2024-03-07T03:42:03.077404+0000 mgr.y (mgr.34101) 843 : cluster 0 pgmap v1405: 105 pgs: 105 active+clean; 83 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.1 MiB/s wr, 25 op/s 2024-03-07T03:42:06.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:05 smithi148 bash[79819]: cluster 2024-03-07T03:42:04.749236+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2024-03-07T03:42:06.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:05 smithi148 bash[82222]: cluster 2024-03-07T03:42:04.749236+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2024-03-07T03:42:06.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:05 smithi195 bash[66542]: cluster 2024-03-07T03:42:04.749236+0000 mon.a (mon.0) 1671 : cluster 0 osdmap e944: 8 total, 8 up, 8 in 2024-03-07T03:42:07.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:06 smithi148 bash[79819]: cluster 2024-03-07T03:42:05.077891+0000 mgr.y (mgr.34101) 844 : cluster 0 pgmap v1407: 105 pgs: 105 active+clean; 95 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-03-07T03:42:07.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:06 smithi148 bash[82222]: cluster 2024-03-07T03:42:05.077891+0000 mgr.y (mgr.34101) 844 : cluster 0 pgmap v1407: 105 pgs: 105 active+clean; 95 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-03-07T03:42:07.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:06 smithi195 bash[66542]: cluster 2024-03-07T03:42:05.077891+0000 mgr.y (mgr.34101) 844 : cluster 0 pgmap v1407: 105 pgs: 105 active+clean; 95 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 6.8 MiB/s rd, 3.2 MiB/s wr, 17 op/s 2024-03-07T03:42:07.543 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:42:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:42:07] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2024-03-07T03:42:09.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:08 smithi148 bash[79819]: cluster 2024-03-07T03:42:07.078790+0000 mgr.y (mgr.34101) 845 : cluster 0 pgmap v1408: 105 pgs: 105 active+clean; 107 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-03-07T03:42:09.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:08 smithi148 bash[82222]: cluster 2024-03-07T03:42:07.078790+0000 mgr.y (mgr.34101) 845 : cluster 0 pgmap v1408: 105 pgs: 105 active+clean; 107 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-03-07T03:42:09.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:08 smithi195 bash[66542]: cluster 2024-03-07T03:42:07.078790+0000 mgr.y (mgr.34101) 845 : cluster 0 pgmap v1408: 105 pgs: 105 active+clean; 107 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 2.4 MiB/s wr, 15 op/s 2024-03-07T03:42:11.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:10 smithi148 bash[79819]: cluster 2024-03-07T03:42:09.079480+0000 mgr.y (mgr.34101) 846 : cluster 0 pgmap v1409: 105 pgs: 105 active+clean; 124 MiB data, 976 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:42:11.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:10 smithi148 bash[82222]: cluster 2024-03-07T03:42:09.079480+0000 mgr.y (mgr.34101) 846 : cluster 0 pgmap v1409: 105 pgs: 105 active+clean; 124 MiB data, 976 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:42:11.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:10 smithi195 bash[66542]: cluster 2024-03-07T03:42:09.079480+0000 mgr.y (mgr.34101) 846 : cluster 0 pgmap v1409: 105 pgs: 105 active+clean; 124 MiB data, 976 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:42:11.530 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/5.Grow (17905 ms) 2024-03-07T03:42:11.530 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/5.Shrink 2024-03-07T03:42:11.530 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:42:12.566 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:42:12 smithi195 bash[59134]: ts=2024-03-07T03:42:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:13.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:12 smithi148 bash[79819]: cluster 2024-03-07T03:42:11.080061+0000 mgr.y (mgr.34101) 847 : cluster 0 pgmap v1410: 105 pgs: 105 active+clean; 132 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.9 MiB/s wr, 42 op/s 2024-03-07T03:42:13.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:12 smithi148 bash[79819]: audit 2024-03-07T03:42:12.341112+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:13.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:12 smithi148 bash[82222]: cluster 2024-03-07T03:42:11.080061+0000 mgr.y (mgr.34101) 847 : cluster 0 pgmap v1410: 105 pgs: 105 active+clean; 132 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.9 MiB/s wr, 42 op/s 2024-03-07T03:42:13.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:12 smithi148 bash[82222]: audit 2024-03-07T03:42:12.341112+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:13.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:12 smithi195 bash[66542]: cluster 2024-03-07T03:42:11.080061+0000 mgr.y (mgr.34101) 847 : cluster 0 pgmap v1410: 105 pgs: 105 active+clean; 132 MiB data, 988 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.9 MiB/s wr, 42 op/s 2024-03-07T03:42:13.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:12 smithi195 bash[66542]: audit 2024-03-07T03:42:12.341112+0000 mon.a (mon.0) 1672 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:15.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:14 smithi148 bash[79819]: cluster 2024-03-07T03:42:13.080577+0000 mgr.y (mgr.34101) 848 : cluster 0 pgmap v1411: 105 pgs: 105 active+clean; 133 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.0 MiB/s wr, 53 op/s 2024-03-07T03:42:15.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:14 smithi148 bash[82222]: cluster 2024-03-07T03:42:13.080577+0000 mgr.y (mgr.34101) 848 : cluster 0 pgmap v1411: 105 pgs: 105 active+clean; 133 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.0 MiB/s wr, 53 op/s 2024-03-07T03:42:15.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:14 smithi195 bash[66542]: cluster 2024-03-07T03:42:13.080577+0000 mgr.y (mgr.34101) 848 : cluster 0 pgmap v1411: 105 pgs: 105 active+clean; 133 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.0 MiB/s wr, 53 op/s 2024-03-07T03:42:17.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:16 smithi148 bash[79819]: cluster 2024-03-07T03:42:15.081337+0000 mgr.y (mgr.34101) 849 : cluster 0 pgmap v1412: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.8 MiB/s wr, 64 op/s 2024-03-07T03:42:17.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:16 smithi148 bash[82222]: cluster 2024-03-07T03:42:15.081337+0000 mgr.y (mgr.34101) 849 : cluster 0 pgmap v1412: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.8 MiB/s wr, 64 op/s 2024-03-07T03:42:17.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:16 smithi195 bash[66542]: cluster 2024-03-07T03:42:15.081337+0000 mgr.y (mgr.34101) 849 : cluster 0 pgmap v1412: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.8 MiB/s wr, 64 op/s 2024-03-07T03:42:17.543 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:42:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:42:17] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-03-07T03:42:19.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:18 smithi148 bash[79819]: cluster 2024-03-07T03:42:17.081932+0000 mgr.y (mgr.34101) 850 : cluster 0 pgmap v1413: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.8 MiB/s wr, 57 op/s 2024-03-07T03:42:19.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:18 smithi148 bash[82222]: cluster 2024-03-07T03:42:17.081932+0000 mgr.y (mgr.34101) 850 : cluster 0 pgmap v1413: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.8 MiB/s wr, 57 op/s 2024-03-07T03:42:19.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:18 smithi195 bash[66542]: cluster 2024-03-07T03:42:17.081932+0000 mgr.y (mgr.34101) 850 : cluster 0 pgmap v1413: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.8 MiB/s wr, 57 op/s 2024-03-07T03:42:21.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:20 smithi148 bash[79819]: cluster 2024-03-07T03:42:19.083065+0000 mgr.y (mgr.34101) 851 : cluster 0 pgmap v1414: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 46 op/s 2024-03-07T03:42:21.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:20 smithi148 bash[79819]: cluster 2024-03-07T03:42:19.797855+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2024-03-07T03:42:21.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:20 smithi148 bash[82222]: cluster 2024-03-07T03:42:19.083065+0000 mgr.y (mgr.34101) 851 : cluster 0 pgmap v1414: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 46 op/s 2024-03-07T03:42:21.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:20 smithi148 bash[82222]: cluster 2024-03-07T03:42:19.797855+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2024-03-07T03:42:21.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:20 smithi195 bash[66542]: cluster 2024-03-07T03:42:19.083065+0000 mgr.y (mgr.34101) 851 : cluster 0 pgmap v1414: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 4.8 MiB/s wr, 46 op/s 2024-03-07T03:42:21.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:20 smithi195 bash[66542]: cluster 2024-03-07T03:42:19.797855+0000 mon.a (mon.0) 1673 : cluster 0 osdmap e945: 8 total, 8 up, 8 in 2024-03-07T03:42:22.566 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:42:22 smithi195 bash[59134]: ts=2024-03-07T03:42:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:23.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:22 smithi148 bash[79819]: cluster 2024-03-07T03:42:21.083676+0000 mgr.y (mgr.34101) 852 : cluster 0 pgmap v1416: 105 pgs: 105 active+clean; 169 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 37 op/s 2024-03-07T03:42:23.043 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:22 smithi148 bash[82222]: cluster 2024-03-07T03:42:21.083676+0000 mgr.y (mgr.34101) 852 : cluster 0 pgmap v1416: 105 pgs: 105 active+clean; 169 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 37 op/s 2024-03-07T03:42:23.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:22 smithi195 bash[66542]: cluster 2024-03-07T03:42:21.083676+0000 mgr.y (mgr.34101) 852 : cluster 0 pgmap v1416: 105 pgs: 105 active+clean; 169 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.7 MiB/s wr, 37 op/s 2024-03-07T03:42:24.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:23 smithi148 bash[79819]: cluster 2024-03-07T03:42:23.084254+0000 mgr.y (mgr.34101) 853 : cluster 0 pgmap v1417: 105 pgs: 105 active+clean; 173 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 4.0 MiB/s wr, 29 op/s 2024-03-07T03:42:24.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:23 smithi148 bash[82222]: cluster 2024-03-07T03:42:23.084254+0000 mgr.y (mgr.34101) 853 : cluster 0 pgmap v1417: 105 pgs: 105 active+clean; 173 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 4.0 MiB/s wr, 29 op/s 2024-03-07T03:42:24.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:23 smithi195 bash[66542]: cluster 2024-03-07T03:42:23.084254+0000 mgr.y (mgr.34101) 853 : cluster 0 pgmap v1417: 105 pgs: 105 active+clean; 173 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.9 MiB/s rd, 4.0 MiB/s wr, 29 op/s 2024-03-07T03:42:25.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:24 smithi148 bash[79819]: audit 2024-03-07T03:42:24.246753+0000 mon.a (mon.0) 1674 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:42:25.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:24 smithi148 bash[79819]: audit 2024-03-07T03:42:24.770184+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:25.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:24 smithi148 bash[79819]: audit 2024-03-07T03:42:24.777759+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:25.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:24 smithi148 bash[82222]: audit 2024-03-07T03:42:24.246753+0000 mon.a (mon.0) 1674 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:42:25.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:24 smithi148 bash[82222]: audit 2024-03-07T03:42:24.770184+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:25.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:24 smithi148 bash[82222]: audit 2024-03-07T03:42:24.777759+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:25.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:24 smithi195 bash[66542]: audit 2024-03-07T03:42:24.246753+0000 mon.a (mon.0) 1674 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:42:25.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:24 smithi195 bash[66542]: audit 2024-03-07T03:42:24.770184+0000 mon.a (mon.0) 1675 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:25.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:24 smithi195 bash[66542]: audit 2024-03-07T03:42:24.777759+0000 mon.a (mon.0) 1676 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:25.927 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/5.Shrink (14397 ms) 2024-03-07T03:42:25.927 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/5.ShrinkToOne 2024-03-07T03:42:25.927 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:42:26.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:25 smithi148 bash[79819]: cluster 2024-03-07T03:42:25.085017+0000 mgr.y (mgr.34101) 854 : cluster 0 pgmap v1418: 105 pgs: 105 active+clean; 215 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.0 MiB/s wr, 46 op/s 2024-03-07T03:42:26.293 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:25 smithi148 bash[82222]: cluster 2024-03-07T03:42:25.085017+0000 mgr.y (mgr.34101) 854 : cluster 0 pgmap v1418: 105 pgs: 105 active+clean; 215 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.0 MiB/s wr, 46 op/s 2024-03-07T03:42:26.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:25 smithi195 bash[66542]: cluster 2024-03-07T03:42:25.085017+0000 mgr.y (mgr.34101) 854 : cluster 0 pgmap v1418: 105 pgs: 105 active+clean; 215 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.0 MiB/s wr, 46 op/s 2024-03-07T03:42:27.543 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:42:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:42:27] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-03-07T03:42:28.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:28 smithi148 bash[79819]: cluster 2024-03-07T03:42:27.085838+0000 mgr.y (mgr.34101) 855 : cluster 0 pgmap v1419: 105 pgs: 105 active+clean; 227 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 46 op/s 2024-03-07T03:42:28.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:28 smithi148 bash[79819]: audit 2024-03-07T03:42:27.340408+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:28.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:28 smithi148 bash[82222]: cluster 2024-03-07T03:42:27.085838+0000 mgr.y (mgr.34101) 855 : cluster 0 pgmap v1419: 105 pgs: 105 active+clean; 227 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 46 op/s 2024-03-07T03:42:28.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:28 smithi148 bash[82222]: audit 2024-03-07T03:42:27.340408+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:28.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:28 smithi195 bash[66542]: cluster 2024-03-07T03:42:27.085838+0000 mgr.y (mgr.34101) 855 : cluster 0 pgmap v1419: 105 pgs: 105 active+clean; 227 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 6.2 MiB/s wr, 46 op/s 2024-03-07T03:42:28.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:28 smithi195 bash[66542]: audit 2024-03-07T03:42:27.340408+0000 mon.a (mon.0) 1677 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:30.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:30 smithi148 bash[79819]: cluster 2024-03-07T03:42:29.086661+0000 mgr.y (mgr.34101) 856 : cluster 0 pgmap v1420: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 71 op/s 2024-03-07T03:42:30.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:30 smithi148 bash[79819]: audit 2024-03-07T03:42:30.129360+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:30.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:30 smithi148 bash[79819]: audit 2024-03-07T03:42:30.135998+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:30.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:30 smithi148 bash[82222]: cluster 2024-03-07T03:42:29.086661+0000 mgr.y (mgr.34101) 856 : cluster 0 pgmap v1420: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 71 op/s 2024-03-07T03:42:30.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:30 smithi148 bash[82222]: audit 2024-03-07T03:42:30.129360+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:30.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:30 smithi148 bash[82222]: audit 2024-03-07T03:42:30.135998+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:30.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:30 smithi195 bash[66542]: cluster 2024-03-07T03:42:29.086661+0000 mgr.y (mgr.34101) 856 : cluster 0 pgmap v1420: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 71 op/s 2024-03-07T03:42:30.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:30 smithi195 bash[66542]: audit 2024-03-07T03:42:30.129360+0000 mon.a (mon.0) 1678 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:30.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:30 smithi195 bash[66542]: audit 2024-03-07T03:42:30.135998+0000 mon.a (mon.0) 1679 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[79819]: audit 2024-03-07T03:42:30.828011+0000 mon.a (mon.0) 1680 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[79819]: audit 2024-03-07T03:42:30.835274+0000 mon.a (mon.0) 1681 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[79819]: cluster 2024-03-07T03:42:31.087269+0000 mgr.y (mgr.34101) 857 : cluster 0 pgmap v1421: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.3 MiB/s wr, 63 op/s 2024-03-07T03:42:32.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[79819]: audit 2024-03-07T03:42:31.404981+0000 mon.a (mon.0) 1682 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:42:32.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[79819]: audit 2024-03-07T03:42:31.406085+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:42:32.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[79819]: audit 2024-03-07T03:42:31.413377+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[82222]: audit 2024-03-07T03:42:30.828011+0000 mon.a (mon.0) 1680 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[82222]: audit 2024-03-07T03:42:30.835274+0000 mon.a (mon.0) 1681 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[82222]: cluster 2024-03-07T03:42:31.087269+0000 mgr.y (mgr.34101) 857 : cluster 0 pgmap v1421: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.3 MiB/s wr, 63 op/s 2024-03-07T03:42:32.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[82222]: audit 2024-03-07T03:42:31.404981+0000 mon.a (mon.0) 1682 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:42:32.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[82222]: audit 2024-03-07T03:42:31.406085+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:42:32.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:31 smithi148 bash[82222]: audit 2024-03-07T03:42:31.413377+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.307 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:31 smithi195 bash[66542]: audit 2024-03-07T03:42:30.828011+0000 mon.a (mon.0) 1680 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:31 smithi195 bash[66542]: audit 2024-03-07T03:42:30.835274+0000 mon.a (mon.0) 1681 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:31 smithi195 bash[66542]: cluster 2024-03-07T03:42:31.087269+0000 mgr.y (mgr.34101) 857 : cluster 0 pgmap v1421: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.3 MiB/s wr, 63 op/s 2024-03-07T03:42:32.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:31 smithi195 bash[66542]: audit 2024-03-07T03:42:31.404981+0000 mon.a (mon.0) 1682 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:42:32.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:31 smithi195 bash[66542]: audit 2024-03-07T03:42:31.406085+0000 mon.a (mon.0) 1683 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:42:32.308 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:31 smithi195 bash[66542]: audit 2024-03-07T03:42:31.413377+0000 mon.a (mon.0) 1684 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:42:32.567 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:42:32 smithi195 bash[59134]: ts=2024-03-07T03:42:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:34.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:33 smithi148 bash[79819]: cluster 2024-03-07T03:42:32.850311+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2024-03-07T03:42:34.293 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:33 smithi148 bash[79819]: cluster 2024-03-07T03:42:33.087764+0000 mgr.y (mgr.34101) 858 : cluster 0 pgmap v1423: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.4 MiB/s wr, 58 op/s 2024-03-07T03:42:34.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:33 smithi148 bash[82222]: cluster 2024-03-07T03:42:32.850311+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2024-03-07T03:42:34.294 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:33 smithi148 bash[82222]: cluster 2024-03-07T03:42:33.087764+0000 mgr.y (mgr.34101) 858 : cluster 0 pgmap v1423: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.4 MiB/s wr, 58 op/s 2024-03-07T03:42:34.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:33 smithi195 bash[66542]: cluster 2024-03-07T03:42:32.850311+0000 mon.a (mon.0) 1685 : cluster 0 osdmap e946: 8 total, 8 up, 8 in 2024-03-07T03:42:34.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:33 smithi195 bash[66542]: cluster 2024-03-07T03:42:33.087764+0000 mgr.y (mgr.34101) 858 : cluster 0 pgmap v1423: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.4 MiB/s wr, 58 op/s 2024-03-07T03:42:36.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:36 smithi148 bash[79819]: cluster 2024-03-07T03:42:35.088701+0000 mgr.y (mgr.34101) 859 : cluster 0 pgmap v1424: 105 pgs: 105 active+clean; 271 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.6 MiB/s wr, 66 op/s 2024-03-07T03:42:36.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:36 smithi148 bash[82222]: cluster 2024-03-07T03:42:35.088701+0000 mgr.y (mgr.34101) 859 : cluster 0 pgmap v1424: 105 pgs: 105 active+clean; 271 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.6 MiB/s wr, 66 op/s 2024-03-07T03:42:36.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:36 smithi195 bash[66542]: cluster 2024-03-07T03:42:35.088701+0000 mgr.y (mgr.34101) 859 : cluster 0 pgmap v1424: 105 pgs: 105 active+clean; 271 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 5.6 MiB/s wr, 66 op/s 2024-03-07T03:42:37.543 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:42:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:42:37] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-03-07T03:42:37.879 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/5.ShrinkToOne (11952 ms) 2024-03-07T03:42:37.879 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/5.ShrinkToOneAfterSnapshot 2024-03-07T03:42:37.879 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:42:38.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:38 smithi148 bash[79819]: cluster 2024-03-07T03:42:37.089322+0000 mgr.y (mgr.34101) 860 : cluster 0 pgmap v1425: 105 pgs: 105 active+clean; 296 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2024-03-07T03:42:38.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:38 smithi148 bash[82222]: cluster 2024-03-07T03:42:37.089322+0000 mgr.y (mgr.34101) 860 : cluster 0 pgmap v1425: 105 pgs: 105 active+clean; 296 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2024-03-07T03:42:38.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:38 smithi195 bash[66542]: cluster 2024-03-07T03:42:37.089322+0000 mgr.y (mgr.34101) 860 : cluster 0 pgmap v1425: 105 pgs: 105 active+clean; 296 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2024-03-07T03:42:40.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:40 smithi148 bash[79819]: cluster 2024-03-07T03:42:39.090169+0000 mgr.y (mgr.34101) 861 : cluster 0 pgmap v1426: 105 pgs: 105 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 5.3 MiB/s wr, 59 op/s 2024-03-07T03:42:40.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:40 smithi148 bash[82222]: cluster 2024-03-07T03:42:39.090169+0000 mgr.y (mgr.34101) 861 : cluster 0 pgmap v1426: 105 pgs: 105 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 5.3 MiB/s wr, 59 op/s 2024-03-07T03:42:40.566 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:40 smithi195 bash[66542]: cluster 2024-03-07T03:42:39.090169+0000 mgr.y (mgr.34101) 861 : cluster 0 pgmap v1426: 105 pgs: 105 active+clean; 300 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 5.3 MiB/s wr, 59 op/s 2024-03-07T03:42:42.543 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:42 smithi148 bash[79819]: cluster 2024-03-07T03:42:41.090804+0000 mgr.y (mgr.34101) 862 : cluster 0 pgmap v1427: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 61 op/s 2024-03-07T03:42:42.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:42 smithi148 bash[82222]: cluster 2024-03-07T03:42:41.090804+0000 mgr.y (mgr.34101) 862 : cluster 0 pgmap v1427: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 61 op/s 2024-03-07T03:42:42.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:42 smithi195 bash[66542]: cluster 2024-03-07T03:42:41.090804+0000 mgr.y (mgr.34101) 862 : cluster 0 pgmap v1427: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 61 op/s 2024-03-07T03:42:42.567 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:42:42 smithi195 bash[59134]: ts=2024-03-07T03:42:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:43.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:43 smithi148 bash[79819]: audit 2024-03-07T03:42:42.341076+0000 mon.a (mon.0) 1686 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:43.544 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:43 smithi148 bash[82222]: audit 2024-03-07T03:42:42.341076+0000 mon.a (mon.0) 1686 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:43.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:43 smithi195 bash[66542]: audit 2024-03-07T03:42:42.341076+0000 mon.a (mon.0) 1686 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:45.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:44 smithi148 bash[79819]: cluster 2024-03-07T03:42:43.091366+0000 mgr.y (mgr.34101) 863 : cluster 0 pgmap v1428: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.3 MiB/s wr, 60 op/s 2024-03-07T03:42:45.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:44 smithi148 bash[82222]: cluster 2024-03-07T03:42:43.091366+0000 mgr.y (mgr.34101) 863 : cluster 0 pgmap v1428: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.3 MiB/s wr, 60 op/s 2024-03-07T03:42:45.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:44 smithi195 bash[66542]: cluster 2024-03-07T03:42:43.091366+0000 mgr.y (mgr.34101) 863 : cluster 0 pgmap v1428: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.3 MiB/s wr, 60 op/s 2024-03-07T03:42:47.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:46 smithi148 bash[79819]: cluster 2024-03-07T03:42:45.092216+0000 mgr.y (mgr.34101) 864 : cluster 0 pgmap v1429: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.4 MiB/s wr, 51 op/s 2024-03-07T03:42:47.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:46 smithi148 bash[82222]: cluster 2024-03-07T03:42:45.092216+0000 mgr.y (mgr.34101) 864 : cluster 0 pgmap v1429: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.4 MiB/s wr, 51 op/s 2024-03-07T03:42:47.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:46 smithi195 bash[66542]: cluster 2024-03-07T03:42:45.092216+0000 mgr.y (mgr.34101) 864 : cluster 0 pgmap v1429: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.4 MiB/s wr, 51 op/s 2024-03-07T03:42:47.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:42:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:42:47] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-03-07T03:42:48.043 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:47 smithi148 bash[79819]: cluster 2024-03-07T03:42:46.671760+0000 mon.a (mon.0) 1687 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2024-03-07T03:42:48.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:47 smithi148 bash[82222]: cluster 2024-03-07T03:42:46.671760+0000 mon.a (mon.0) 1687 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2024-03-07T03:42:48.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:47 smithi195 bash[66542]: cluster 2024-03-07T03:42:46.671760+0000 mon.a (mon.0) 1687 : cluster 0 osdmap e947: 8 total, 8 up, 8 in 2024-03-07T03:42:49.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:48 smithi148 bash[79819]: cluster 2024-03-07T03:42:47.092705+0000 mgr.y (mgr.34101) 865 : cluster 0 pgmap v1431: 105 pgs: 105 active+clean; 316 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-03-07T03:42:49.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:48 smithi148 bash[82222]: cluster 2024-03-07T03:42:47.092705+0000 mgr.y (mgr.34101) 865 : cluster 0 pgmap v1431: 105 pgs: 105 active+clean; 316 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-03-07T03:42:49.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:48 smithi195 bash[66542]: cluster 2024-03-07T03:42:47.092705+0000 mgr.y (mgr.34101) 865 : cluster 0 pgmap v1431: 105 pgs: 105 active+clean; 316 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 2.4 MiB/s wr, 19 op/s 2024-03-07T03:42:51.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:50 smithi148 bash[79819]: cluster 2024-03-07T03:42:49.093478+0000 mgr.y (mgr.34101) 866 : cluster 0 pgmap v1432: 105 pgs: 105 active+clean; 345 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 4.9 MiB/s wr, 35 op/s 2024-03-07T03:42:51.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:50 smithi148 bash[82222]: cluster 2024-03-07T03:42:49.093478+0000 mgr.y (mgr.34101) 866 : cluster 0 pgmap v1432: 105 pgs: 105 active+clean; 345 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 4.9 MiB/s wr, 35 op/s 2024-03-07T03:42:51.066 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:50 smithi195 bash[66542]: cluster 2024-03-07T03:42:49.093478+0000 mgr.y (mgr.34101) 866 : cluster 0 pgmap v1432: 105 pgs: 105 active+clean; 345 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 4.9 MiB/s wr, 35 op/s 2024-03-07T03:42:52.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:51 smithi148 bash[79819]: cluster 2024-03-07T03:42:50.699434+0000 mon.a (mon.0) 1688 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2024-03-07T03:42:52.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:51 smithi148 bash[82222]: cluster 2024-03-07T03:42:50.699434+0000 mon.a (mon.0) 1688 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2024-03-07T03:42:52.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:51 smithi195 bash[66542]: cluster 2024-03-07T03:42:50.699434+0000 mon.a (mon.0) 1688 : cluster 0 osdmap e948: 8 total, 8 up, 8 in 2024-03-07T03:42:52.567 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:42:52 smithi195 bash[59134]: ts=2024-03-07T03:42:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:42:52.990 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/5.ShrinkToOneAfterSnapshot (15110 ms) 2024-03-07T03:42:52.990 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/5.MinOverlap 2024-03-07T03:42:52.990 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:42:53.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:52 smithi148 bash[79819]: cluster 2024-03-07T03:42:51.093866+0000 mgr.y (mgr.34101) 867 : cluster 0 pgmap v1434: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 5.1 MiB/s wr, 50 op/s 2024-03-07T03:42:53.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:52 smithi148 bash[82222]: cluster 2024-03-07T03:42:51.093866+0000 mgr.y (mgr.34101) 867 : cluster 0 pgmap v1434: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 5.1 MiB/s wr, 50 op/s 2024-03-07T03:42:53.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:52 smithi195 bash[66542]: cluster 2024-03-07T03:42:51.093866+0000 mgr.y (mgr.34101) 867 : cluster 0 pgmap v1434: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 109 KiB/s rd, 5.1 MiB/s wr, 50 op/s 2024-03-07T03:42:55.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:54 smithi148 bash[79819]: cluster 2024-03-07T03:42:53.094573+0000 mgr.y (mgr.34101) 868 : cluster 0 pgmap v1435: 105 pgs: 105 active+clean; 357 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 6.1 MiB/s wr, 54 op/s 2024-03-07T03:42:55.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:54 smithi148 bash[82222]: cluster 2024-03-07T03:42:53.094573+0000 mgr.y (mgr.34101) 868 : cluster 0 pgmap v1435: 105 pgs: 105 active+clean; 357 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 6.1 MiB/s wr, 54 op/s 2024-03-07T03:42:55.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:54 smithi195 bash[66542]: cluster 2024-03-07T03:42:53.094573+0000 mgr.y (mgr.34101) 868 : cluster 0 pgmap v1435: 105 pgs: 105 active+clean; 357 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 6.1 MiB/s wr, 54 op/s 2024-03-07T03:42:57.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:56 smithi148 bash[79819]: cluster 2024-03-07T03:42:55.095406+0000 mgr.y (mgr.34101) 869 : cluster 0 pgmap v1436: 105 pgs: 105 active+clean; 426 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 13 MiB/s wr, 98 op/s 2024-03-07T03:42:57.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:56 smithi148 bash[82222]: cluster 2024-03-07T03:42:55.095406+0000 mgr.y (mgr.34101) 869 : cluster 0 pgmap v1436: 105 pgs: 105 active+clean; 426 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 13 MiB/s wr, 98 op/s 2024-03-07T03:42:57.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:56 smithi195 bash[66542]: cluster 2024-03-07T03:42:55.095406+0000 mgr.y (mgr.34101) 869 : cluster 0 pgmap v1436: 105 pgs: 105 active+clean; 426 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6.0 MiB/s rd, 13 MiB/s wr, 98 op/s 2024-03-07T03:42:57.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:42:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:42:57] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-03-07T03:42:58.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:57 smithi148 bash[79819]: audit 2024-03-07T03:42:57.341712+0000 mon.a (mon.0) 1689 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:58.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:57 smithi148 bash[82222]: audit 2024-03-07T03:42:57.341712+0000 mon.a (mon.0) 1689 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:58.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:57 smithi195 bash[66542]: audit 2024-03-07T03:42:57.341712+0000 mon.a (mon.0) 1689 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:42:59.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:42:58 smithi148 bash[79819]: cluster 2024-03-07T03:42:57.096015+0000 mgr.y (mgr.34101) 870 : cluster 0 pgmap v1437: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 12 MiB/s wr, 83 op/s 2024-03-07T03:42:59.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:42:58 smithi148 bash[82222]: cluster 2024-03-07T03:42:57.096015+0000 mgr.y (mgr.34101) 870 : cluster 0 pgmap v1437: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 12 MiB/s wr, 83 op/s 2024-03-07T03:42:59.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:42:58 smithi195 bash[66542]: cluster 2024-03-07T03:42:57.096015+0000 mgr.y (mgr.34101) 870 : cluster 0 pgmap v1437: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 12 MiB/s wr, 83 op/s 2024-03-07T03:43:01.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:00 smithi148 bash[79819]: cluster 2024-03-07T03:42:59.096802+0000 mgr.y (mgr.34101) 871 : cluster 0 pgmap v1438: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.9 MiB/s wr, 51 op/s 2024-03-07T03:43:01.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:00 smithi148 bash[82222]: cluster 2024-03-07T03:42:59.096802+0000 mgr.y (mgr.34101) 871 : cluster 0 pgmap v1438: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.9 MiB/s wr, 51 op/s 2024-03-07T03:43:01.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:00 smithi195 bash[66542]: cluster 2024-03-07T03:42:59.096802+0000 mgr.y (mgr.34101) 871 : cluster 0 pgmap v1438: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.9 MiB/s wr, 51 op/s 2024-03-07T03:43:02.567 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:43:02 smithi195 bash[59134]: ts=2024-03-07T03:43:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:03.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:02 smithi148 bash[79819]: cluster 2024-03-07T03:43:01.097586+0000 mgr.y (mgr.34101) 872 : cluster 0 pgmap v1439: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 8.2 MiB/s wr, 42 op/s 2024-03-07T03:43:03.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:02 smithi148 bash[79819]: cluster 2024-03-07T03:43:01.762027+0000 mon.a (mon.0) 1690 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2024-03-07T03:43:03.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:02 smithi148 bash[82222]: cluster 2024-03-07T03:43:01.097586+0000 mgr.y (mgr.34101) 872 : cluster 0 pgmap v1439: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 8.2 MiB/s wr, 42 op/s 2024-03-07T03:43:03.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:02 smithi148 bash[82222]: cluster 2024-03-07T03:43:01.762027+0000 mon.a (mon.0) 1690 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2024-03-07T03:43:03.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:02 smithi195 bash[66542]: cluster 2024-03-07T03:43:01.097586+0000 mgr.y (mgr.34101) 872 : cluster 0 pgmap v1439: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.5 MiB/s rd, 8.2 MiB/s wr, 42 op/s 2024-03-07T03:43:03.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:02 smithi195 bash[66542]: cluster 2024-03-07T03:43:01.762027+0000 mon.a (mon.0) 1690 : cluster 0 osdmap e949: 8 total, 8 up, 8 in 2024-03-07T03:43:05.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:04 smithi148 bash[79819]: cluster 2024-03-07T03:43:03.098039+0000 mgr.y (mgr.34101) 873 : cluster 0 pgmap v1441: 105 pgs: 105 active+clean; 443 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 8.6 MiB/s wr, 41 op/s 2024-03-07T03:43:05.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:04 smithi148 bash[82222]: cluster 2024-03-07T03:43:03.098039+0000 mgr.y (mgr.34101) 873 : cluster 0 pgmap v1441: 105 pgs: 105 active+clean; 443 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 8.6 MiB/s wr, 41 op/s 2024-03-07T03:43:05.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:04 smithi195 bash[66542]: cluster 2024-03-07T03:43:03.098039+0000 mgr.y (mgr.34101) 873 : cluster 0 pgmap v1441: 105 pgs: 105 active+clean; 443 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.5 MiB/s rd, 8.6 MiB/s wr, 41 op/s 2024-03-07T03:43:07.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:06 smithi148 bash[79819]: cluster 2024-03-07T03:43:05.098809+0000 mgr.y (mgr.34101) 874 : cluster 0 pgmap v1442: 105 pgs: 105 active+clean; 471 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 842 KiB/s rd, 4.5 MiB/s wr, 32 op/s 2024-03-07T03:43:07.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:06 smithi148 bash[82222]: cluster 2024-03-07T03:43:05.098809+0000 mgr.y (mgr.34101) 874 : cluster 0 pgmap v1442: 105 pgs: 105 active+clean; 471 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 842 KiB/s rd, 4.5 MiB/s wr, 32 op/s 2024-03-07T03:43:07.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:06 smithi195 bash[66542]: cluster 2024-03-07T03:43:05.098809+0000 mgr.y (mgr.34101) 874 : cluster 0 pgmap v1442: 105 pgs: 105 active+clean; 471 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 842 KiB/s rd, 4.5 MiB/s wr, 32 op/s 2024-03-07T03:43:07.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:43:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:43:07] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-03-07T03:43:07.665 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/5.MinOverlap (14675 ms) 2024-03-07T03:43:07.665 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/5.ZeroOverlap 2024-03-07T03:43:07.665 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:43:09.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:08 smithi148 bash[79819]: cluster 2024-03-07T03:43:07.099446+0000 mgr.y (mgr.34101) 875 : cluster 0 pgmap v1443: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 915 KiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:43:09.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:08 smithi148 bash[82222]: cluster 2024-03-07T03:43:07.099446+0000 mgr.y (mgr.34101) 875 : cluster 0 pgmap v1443: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 915 KiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:43:09.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:08 smithi195 bash[66542]: cluster 2024-03-07T03:43:07.099446+0000 mgr.y (mgr.34101) 875 : cluster 0 pgmap v1443: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 915 KiB/s rd, 4.1 MiB/s wr, 51 op/s 2024-03-07T03:43:11.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:10 smithi148 bash[79819]: cluster 2024-03-07T03:43:09.100345+0000 mgr.y (mgr.34101) 876 : cluster 0 pgmap v1444: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:43:11.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:10 smithi148 bash[82222]: cluster 2024-03-07T03:43:09.100345+0000 mgr.y (mgr.34101) 876 : cluster 0 pgmap v1444: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:43:11.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:10 smithi195 bash[66542]: cluster 2024-03-07T03:43:09.100345+0000 mgr.y (mgr.34101) 876 : cluster 0 pgmap v1444: 105 pgs: 105 active+clean; 491 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-03-07T03:43:12.567 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:43:12 smithi195 bash[59134]: ts=2024-03-07T03:43:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:13.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:12 smithi148 bash[79819]: cluster 2024-03-07T03:43:11.100960+0000 mgr.y (mgr.34101) 877 : cluster 0 pgmap v1445: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 63 op/s 2024-03-07T03:43:13.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:12 smithi148 bash[79819]: audit 2024-03-07T03:43:12.341884+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:13.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:12 smithi148 bash[82222]: cluster 2024-03-07T03:43:11.100960+0000 mgr.y (mgr.34101) 877 : cluster 0 pgmap v1445: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 63 op/s 2024-03-07T03:43:13.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:12 smithi148 bash[82222]: audit 2024-03-07T03:43:12.341884+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:13.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:12 smithi195 bash[66542]: cluster 2024-03-07T03:43:11.100960+0000 mgr.y (mgr.34101) 877 : cluster 0 pgmap v1445: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 63 op/s 2024-03-07T03:43:13.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:12 smithi195 bash[66542]: audit 2024-03-07T03:43:12.341884+0000 mon.a (mon.0) 1691 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:15.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:14 smithi148 bash[79819]: cluster 2024-03-07T03:43:13.101488+0000 mgr.y (mgr.34101) 878 : cluster 0 pgmap v1446: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.9 MiB/s wr, 56 op/s 2024-03-07T03:43:15.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:14 smithi148 bash[82222]: cluster 2024-03-07T03:43:13.101488+0000 mgr.y (mgr.34101) 878 : cluster 0 pgmap v1446: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.9 MiB/s wr, 56 op/s 2024-03-07T03:43:15.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:14 smithi195 bash[66542]: cluster 2024-03-07T03:43:13.101488+0000 mgr.y (mgr.34101) 878 : cluster 0 pgmap v1446: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.9 MiB/s wr, 56 op/s 2024-03-07T03:43:17.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:16 smithi148 bash[79819]: cluster 2024-03-07T03:43:15.102244+0000 mgr.y (mgr.34101) 879 : cluster 0 pgmap v1447: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.7 MiB/s wr, 53 op/s 2024-03-07T03:43:17.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:16 smithi148 bash[82222]: cluster 2024-03-07T03:43:15.102244+0000 mgr.y (mgr.34101) 879 : cluster 0 pgmap v1447: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.7 MiB/s wr, 53 op/s 2024-03-07T03:43:17.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:16 smithi195 bash[66542]: cluster 2024-03-07T03:43:15.102244+0000 mgr.y (mgr.34101) 879 : cluster 0 pgmap v1447: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 4.7 MiB/s wr, 53 op/s 2024-03-07T03:43:17.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:43:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:43:17] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-03-07T03:43:19.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:18 smithi148 bash[79819]: cluster 2024-03-07T03:43:17.103088+0000 mgr.y (mgr.34101) 880 : cluster 0 pgmap v1448: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-03-07T03:43:19.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:18 smithi148 bash[79819]: cluster 2024-03-07T03:43:17.795979+0000 mon.a (mon.0) 1692 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2024-03-07T03:43:19.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:18 smithi148 bash[82222]: cluster 2024-03-07T03:43:17.103088+0000 mgr.y (mgr.34101) 880 : cluster 0 pgmap v1448: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-03-07T03:43:19.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:18 smithi148 bash[82222]: cluster 2024-03-07T03:43:17.795979+0000 mon.a (mon.0) 1692 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2024-03-07T03:43:19.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:18 smithi195 bash[66542]: cluster 2024-03-07T03:43:17.103088+0000 mgr.y (mgr.34101) 880 : cluster 0 pgmap v1448: 105 pgs: 105 active+clean; 495 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.3 MiB/s wr, 27 op/s 2024-03-07T03:43:19.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:18 smithi195 bash[66542]: cluster 2024-03-07T03:43:17.795979+0000 mon.a (mon.0) 1692 : cluster 0 osdmap e950: 8 total, 8 up, 8 in 2024-03-07T03:43:21.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:20 smithi148 bash[79819]: cluster 2024-03-07T03:43:19.103872+0000 mgr.y (mgr.34101) 881 : cluster 0 pgmap v1450: 105 pgs: 105 active+clean; 524 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 878 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:43:21.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:20 smithi148 bash[82222]: cluster 2024-03-07T03:43:19.103872+0000 mgr.y (mgr.34101) 881 : cluster 0 pgmap v1450: 105 pgs: 105 active+clean; 524 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 878 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:43:21.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:20 smithi195 bash[66542]: cluster 2024-03-07T03:43:19.103872+0000 mgr.y (mgr.34101) 881 : cluster 0 pgmap v1450: 105 pgs: 105 active+clean; 524 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 878 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-03-07T03:43:22.567 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:43:22 smithi195 bash[59134]: ts=2024-03-07T03:43:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:23.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:22 smithi148 bash[79819]: cluster 2024-03-07T03:43:21.104505+0000 mgr.y (mgr.34101) 882 : cluster 0 pgmap v1451: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:43:23.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:22 smithi148 bash[82222]: cluster 2024-03-07T03:43:21.104505+0000 mgr.y (mgr.34101) 882 : cluster 0 pgmap v1451: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:43:23.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:22 smithi195 bash[66542]: cluster 2024-03-07T03:43:21.104505+0000 mgr.y (mgr.34101) 882 : cluster 0 pgmap v1451: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 59 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:43:24.361 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/5.ZeroOverlap (16697 ms) 2024-03-07T03:43:24.810 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/5 (109049 ms total) 2024-03-07T03:43:24.810 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:43:24.810 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/6, where TypeParam = LUKS2UnderLUKS2 2024-03-07T03:43:24.862 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/6.Simple 2024-03-07T03:43:25.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:24 smithi195 bash[66542]: cluster 2024-03-07T03:43:23.105069+0000 mgr.y (mgr.34101) 883 : cluster 0 pgmap v1452: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:43:25.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:24 smithi148 bash[79819]: cluster 2024-03-07T03:43:23.105069+0000 mgr.y (mgr.34101) 883 : cluster 0 pgmap v1452: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:43:25.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:24 smithi148 bash[82222]: cluster 2024-03-07T03:43:23.105069+0000 mgr.y (mgr.34101) 883 : cluster 0 pgmap v1452: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 60 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-03-07T03:43:26.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:25 smithi195 bash[66542]: cluster 2024-03-07T03:43:24.807215+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2024-03-07T03:43:26.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:25 smithi148 bash[79819]: cluster 2024-03-07T03:43:24.807215+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2024-03-07T03:43:26.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:25 smithi148 bash[82222]: cluster 2024-03-07T03:43:24.807215+0000 mon.a (mon.0) 1693 : cluster 0 osdmap e951: 8 total, 8 up, 8 in 2024-03-07T03:43:26.835 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:43:27.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:26 smithi195 bash[66542]: cluster 2024-03-07T03:43:25.105621+0000 mgr.y (mgr.34101) 884 : cluster 0 pgmap v1454: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:43:27.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:26 smithi195 bash[66542]: cluster 2024-03-07T03:43:25.834303+0000 mon.a (mon.0) 1694 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2024-03-07T03:43:27.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:26 smithi195 bash[66542]: audit 2024-03-07T03:43:25.838727+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.148:0/3904001059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-42","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:43:27.172 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:26 smithi148 bash[79819]: cluster 2024-03-07T03:43:25.105621+0000 mgr.y (mgr.34101) 884 : cluster 0 pgmap v1454: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:43:27.172 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:26 smithi148 bash[79819]: cluster 2024-03-07T03:43:25.834303+0000 mon.a (mon.0) 1694 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2024-03-07T03:43:27.172 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:26 smithi148 bash[79819]: audit 2024-03-07T03:43:25.838727+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.148:0/3904001059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-42","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:43:27.172 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:26 smithi148 bash[82222]: cluster 2024-03-07T03:43:25.105621+0000 mgr.y (mgr.34101) 884 : cluster 0 pgmap v1454: 73 pgs: 73 active+clean; 582 KiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:43:27.173 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:26 smithi148 bash[82222]: cluster 2024-03-07T03:43:25.834303+0000 mon.a (mon.0) 1694 : cluster 0 osdmap e952: 8 total, 8 up, 8 in 2024-03-07T03:43:27.173 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:26 smithi148 bash[82222]: audit 2024-03-07T03:43:25.838727+0000 mon.a (mon.0) 1695 : audit 1 from='client.? 172.21.15.148:0/3904001059' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-42","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:43:27.544 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:43:27] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[79819]: audit 2024-03-07T03:43:26.830182+0000 mon.a (mon.0) 1696 : audit 1 from='client.? 172.21.15.148:0/3904001059' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[79819]: cluster 2024-03-07T03:43:26.837999+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[79819]: cluster 2024-03-07T03:43:27.106320+0000 mgr.y (mgr.34101) 885 : cluster 0 pgmap v1457: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[79819]: audit 2024-03-07T03:43:27.342849+0000 mon.a (mon.0) 1698 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[82222]: audit 2024-03-07T03:43:26.830182+0000 mon.a (mon.0) 1696 : audit 1 from='client.? 172.21.15.148:0/3904001059' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[82222]: cluster 2024-03-07T03:43:26.837999+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[82222]: cluster 2024-03-07T03:43:27.106320+0000 mgr.y (mgr.34101) 885 : cluster 0 pgmap v1457: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:43:28.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:27 smithi148 bash[82222]: audit 2024-03-07T03:43:27.342849+0000 mon.a (mon.0) 1698 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:28.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:27 smithi195 bash[66542]: audit 2024-03-07T03:43:26.830182+0000 mon.a (mon.0) 1696 : audit 1 from='client.? 172.21.15.148:0/3904001059' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:43:28.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:27 smithi195 bash[66542]: cluster 2024-03-07T03:43:26.837999+0000 mon.a (mon.0) 1697 : cluster 0 osdmap e953: 8 total, 8 up, 8 in 2024-03-07T03:43:28.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:27 smithi195 bash[66542]: cluster 2024-03-07T03:43:27.106320+0000 mgr.y (mgr.34101) 885 : cluster 0 pgmap v1457: 105 pgs: 5 creating+peering, 27 unknown, 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:43:28.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:27 smithi195 bash[66542]: audit 2024-03-07T03:43:27.342849+0000 mon.a (mon.0) 1698 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:29.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:28 smithi148 bash[79819]: cluster 2024-03-07T03:43:27.853261+0000 mon.a (mon.0) 1699 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2024-03-07T03:43:29.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:28 smithi148 bash[82222]: cluster 2024-03-07T03:43:27.853261+0000 mon.a (mon.0) 1699 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2024-03-07T03:43:29.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:28 smithi195 bash[66542]: cluster 2024-03-07T03:43:27.853261+0000 mon.a (mon.0) 1699 : cluster 0 osdmap e954: 8 total, 8 up, 8 in 2024-03-07T03:43:30.294 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:29 smithi148 bash[79819]: cluster 2024-03-07T03:43:28.856273+0000 mon.a (mon.0) 1700 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2024-03-07T03:43:30.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:29 smithi148 bash[79819]: cluster 2024-03-07T03:43:29.107076+0000 mgr.y (mgr.34101) 886 : cluster 0 pgmap v1460: 105 pgs: 5 creating+peering, 11 unknown, 89 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:43:30.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:29 smithi148 bash[82222]: cluster 2024-03-07T03:43:28.856273+0000 mon.a (mon.0) 1700 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2024-03-07T03:43:30.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:29 smithi148 bash[82222]: cluster 2024-03-07T03:43:29.107076+0000 mgr.y (mgr.34101) 886 : cluster 0 pgmap v1460: 105 pgs: 5 creating+peering, 11 unknown, 89 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:43:30.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:29 smithi195 bash[66542]: cluster 2024-03-07T03:43:28.856273+0000 mon.a (mon.0) 1700 : cluster 0 osdmap e955: 8 total, 8 up, 8 in 2024-03-07T03:43:30.317 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:29 smithi195 bash[66542]: cluster 2024-03-07T03:43:29.107076+0000 mgr.y (mgr.34101) 886 : cluster 0 pgmap v1460: 105 pgs: 5 creating+peering, 11 unknown, 89 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:43:32.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:32 smithi148 bash[79819]: cluster 2024-03-07T03:43:31.107858+0000 mgr.y (mgr.34101) 887 : cluster 0 pgmap v1461: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 775 B/s wr, 6 op/s 2024-03-07T03:43:32.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:32 smithi148 bash[79819]: audit 2024-03-07T03:43:31.497986+0000 mon.a (mon.0) 1701 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:43:32.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:32 smithi148 bash[82222]: cluster 2024-03-07T03:43:31.107858+0000 mgr.y (mgr.34101) 887 : cluster 0 pgmap v1461: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 775 B/s wr, 6 op/s 2024-03-07T03:43:32.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:32 smithi148 bash[82222]: audit 2024-03-07T03:43:31.497986+0000 mon.a (mon.0) 1701 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:43:32.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:32 smithi195 bash[66542]: cluster 2024-03-07T03:43:31.107858+0000 mgr.y (mgr.34101) 887 : cluster 0 pgmap v1461: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 775 B/s wr, 6 op/s 2024-03-07T03:43:32.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:32 smithi195 bash[66542]: audit 2024-03-07T03:43:31.497986+0000 mon.a (mon.0) 1701 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:43:32.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:43:32 smithi195 bash[59134]: ts=2024-03-07T03:43:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:34.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:34 smithi148 bash[79819]: cluster 2024-03-07T03:43:33.108462+0000 mgr.y (mgr.34101) 888 : cluster 0 pgmap v1462: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 652 B/s wr, 5 op/s 2024-03-07T03:43:34.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:34 smithi148 bash[82222]: cluster 2024-03-07T03:43:33.108462+0000 mgr.y (mgr.34101) 888 : cluster 0 pgmap v1462: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 652 B/s wr, 5 op/s 2024-03-07T03:43:34.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:34 smithi195 bash[66542]: cluster 2024-03-07T03:43:33.108462+0000 mgr.y (mgr.34101) 888 : cluster 0 pgmap v1462: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 652 B/s wr, 5 op/s 2024-03-07T03:43:36.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:36 smithi148 bash[79819]: cluster 2024-03-07T03:43:35.109249+0000 mgr.y (mgr.34101) 889 : cluster 0 pgmap v1463: 105 pgs: 105 active+clean; 582 KiB data, 622 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:43:36.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:36 smithi148 bash[82222]: cluster 2024-03-07T03:43:35.109249+0000 mgr.y (mgr.34101) 889 : cluster 0 pgmap v1463: 105 pgs: 105 active+clean; 582 KiB data, 622 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:43:36.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:36 smithi195 bash[66542]: cluster 2024-03-07T03:43:35.109249+0000 mgr.y (mgr.34101) 889 : cluster 0 pgmap v1463: 105 pgs: 105 active+clean; 582 KiB data, 622 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-03-07T03:43:37.544 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:37 smithi148 bash[79819]: cluster 2024-03-07T03:43:36.176908+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2024-03-07T03:43:37.545 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:37 smithi148 bash[82222]: cluster 2024-03-07T03:43:36.176908+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2024-03-07T03:43:37.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:43:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:43:37] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.43.0" 2024-03-07T03:43:37.567 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:37 smithi195 bash[66542]: cluster 2024-03-07T03:43:36.176908+0000 mon.a (mon.0) 1702 : cluster 0 osdmap e956: 8 total, 8 up, 8 in 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: cluster 2024-03-07T03:43:37.109702+0000 mgr.y (mgr.34101) 890 : cluster 0 pgmap v1465: 105 pgs: 105 active+clean; 8.6 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 993 KiB/s wr, 6 op/s 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: audit 2024-03-07T03:43:37.337107+0000 mon.a (mon.0) 1703 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: audit 2024-03-07T03:43:37.343348+0000 mon.a (mon.0) 1704 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: audit 2024-03-07T03:43:37.540255+0000 mon.a (mon.0) 1705 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: audit 2024-03-07T03:43:37.549265+0000 mon.a (mon.0) 1706 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: cluster 2024-03-07T03:43:37.671420+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: audit 2024-03-07T03:43:38.074731+0000 mon.a (mon.0) 1708 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:43:38.795 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: audit 2024-03-07T03:43:38.075461+0000 mon.a (mon.0) 1709 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[79819]: audit 2024-03-07T03:43:38.081250+0000 mon.a (mon.0) 1710 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: cluster 2024-03-07T03:43:37.109702+0000 mgr.y (mgr.34101) 890 : cluster 0 pgmap v1465: 105 pgs: 105 active+clean; 8.6 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 993 KiB/s wr, 6 op/s 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: audit 2024-03-07T03:43:37.337107+0000 mon.a (mon.0) 1703 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: audit 2024-03-07T03:43:37.343348+0000 mon.a (mon.0) 1704 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: audit 2024-03-07T03:43:37.540255+0000 mon.a (mon.0) 1705 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: audit 2024-03-07T03:43:37.549265+0000 mon.a (mon.0) 1706 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: cluster 2024-03-07T03:43:37.671420+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: audit 2024-03-07T03:43:38.074731+0000 mon.a (mon.0) 1708 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: audit 2024-03-07T03:43:38.075461+0000 mon.a (mon.0) 1709 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:43:38.796 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:38 smithi148 bash[82222]: audit 2024-03-07T03:43:38.081250+0000 mon.a (mon.0) 1710 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: cluster 2024-03-07T03:43:37.109702+0000 mgr.y (mgr.34101) 890 : cluster 0 pgmap v1465: 105 pgs: 105 active+clean; 8.6 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 993 KiB/s wr, 6 op/s 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: audit 2024-03-07T03:43:37.337107+0000 mon.a (mon.0) 1703 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: audit 2024-03-07T03:43:37.343348+0000 mon.a (mon.0) 1704 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: audit 2024-03-07T03:43:37.540255+0000 mon.a (mon.0) 1705 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: audit 2024-03-07T03:43:37.549265+0000 mon.a (mon.0) 1706 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: cluster 2024-03-07T03:43:37.671420+0000 mon.a (mon.0) 1707 : cluster 0 osdmap e957: 8 total, 8 up, 8 in 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: audit 2024-03-07T03:43:38.074731+0000 mon.a (mon.0) 1708 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: audit 2024-03-07T03:43:38.075461+0000 mon.a (mon.0) 1709 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:43:38.818 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:38 smithi195 bash[66542]: audit 2024-03-07T03:43:38.081250+0000 mon.a (mon.0) 1710 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:40.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:39 smithi148 bash[79819]: audit 2024-03-07T03:43:38.732216+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:40.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:39 smithi148 bash[82222]: audit 2024-03-07T03:43:38.732216+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:40.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:39 smithi195 bash[66542]: audit 2024-03-07T03:43:38.732216+0000 mon.a (mon.0) 1711 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:43:41.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:40 smithi148 bash[79819]: cluster 2024-03-07T03:43:39.110554+0000 mgr.y (mgr.34101) 891 : cluster 0 pgmap v1467: 105 pgs: 105 active+clean; 26 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.1 MiB/s wr, 17 op/s 2024-03-07T03:43:41.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:40 smithi148 bash[82222]: cluster 2024-03-07T03:43:39.110554+0000 mgr.y (mgr.34101) 891 : cluster 0 pgmap v1467: 105 pgs: 105 active+clean; 26 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.1 MiB/s wr, 17 op/s 2024-03-07T03:43:41.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:40 smithi195 bash[66542]: cluster 2024-03-07T03:43:39.110554+0000 mgr.y (mgr.34101) 891 : cluster 0 pgmap v1467: 105 pgs: 105 active+clean; 26 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 9.4 KiB/s rd, 3.1 MiB/s wr, 17 op/s 2024-03-07T03:43:42.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:43:42 smithi195 bash[59134]: ts=2024-03-07T03:43:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:43.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:42 smithi148 bash[79819]: cluster 2024-03-07T03:43:41.111151+0000 mgr.y (mgr.34101) 892 : cluster 0 pgmap v1468: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 5.1 MiB/s wr, 46 op/s 2024-03-07T03:43:43.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:42 smithi148 bash[79819]: audit 2024-03-07T03:43:42.342675+0000 mon.a (mon.0) 1712 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:43.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:42 smithi148 bash[82222]: cluster 2024-03-07T03:43:41.111151+0000 mgr.y (mgr.34101) 892 : cluster 0 pgmap v1468: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 5.1 MiB/s wr, 46 op/s 2024-03-07T03:43:43.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:42 smithi148 bash[82222]: audit 2024-03-07T03:43:42.342675+0000 mon.a (mon.0) 1712 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:43.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:42 smithi195 bash[66542]: cluster 2024-03-07T03:43:41.111151+0000 mgr.y (mgr.34101) 892 : cluster 0 pgmap v1468: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 5.1 MiB/s wr, 46 op/s 2024-03-07T03:43:43.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:42 smithi195 bash[66542]: audit 2024-03-07T03:43:42.342675+0000 mon.a (mon.0) 1712 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:45.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:44 smithi148 bash[79819]: cluster 2024-03-07T03:43:43.111704+0000 mgr.y (mgr.34101) 893 : cluster 0 pgmap v1469: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-03-07T03:43:45.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:44 smithi148 bash[82222]: cluster 2024-03-07T03:43:43.111704+0000 mgr.y (mgr.34101) 893 : cluster 0 pgmap v1469: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-03-07T03:43:45.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:44 smithi195 bash[66542]: cluster 2024-03-07T03:43:43.111704+0000 mgr.y (mgr.34101) 893 : cluster 0 pgmap v1469: 105 pgs: 105 active+clean; 42 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-03-07T03:43:47.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:46 smithi148 bash[79819]: cluster 2024-03-07T03:43:45.112526+0000 mgr.y (mgr.34101) 894 : cluster 0 pgmap v1470: 105 pgs: 105 active+clean; 50 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 5.5 MiB/s wr, 44 op/s 2024-03-07T03:43:47.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:46 smithi148 bash[82222]: cluster 2024-03-07T03:43:45.112526+0000 mgr.y (mgr.34101) 894 : cluster 0 pgmap v1470: 105 pgs: 105 active+clean; 50 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 5.5 MiB/s wr, 44 op/s 2024-03-07T03:43:47.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:46 smithi195 bash[66542]: cluster 2024-03-07T03:43:45.112526+0000 mgr.y (mgr.34101) 894 : cluster 0 pgmap v1470: 105 pgs: 105 active+clean; 50 MiB data, 773 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 5.5 MiB/s wr, 44 op/s 2024-03-07T03:43:47.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:43:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:43:47] "GET /metrics HTTP/1.1" 200 36735 "" "Prometheus/2.43.0" 2024-03-07T03:43:48.569 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/6.Simple (23707 ms) 2024-03-07T03:43:48.570 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/6.Grow 2024-03-07T03:43:48.570 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:43:49.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:48 smithi148 bash[79819]: cluster 2024-03-07T03:43:47.113157+0000 mgr.y (mgr.34101) 895 : cluster 0 pgmap v1471: 105 pgs: 105 active+clean; 58 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 4.9 MiB/s wr, 46 op/s 2024-03-07T03:43:49.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:48 smithi148 bash[82222]: cluster 2024-03-07T03:43:47.113157+0000 mgr.y (mgr.34101) 895 : cluster 0 pgmap v1471: 105 pgs: 105 active+clean; 58 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 4.9 MiB/s wr, 46 op/s 2024-03-07T03:43:49.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:48 smithi195 bash[66542]: cluster 2024-03-07T03:43:47.113157+0000 mgr.y (mgr.34101) 895 : cluster 0 pgmap v1471: 105 pgs: 105 active+clean; 58 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 4.9 MiB/s wr, 46 op/s 2024-03-07T03:43:51.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:50 smithi148 bash[79819]: cluster 2024-03-07T03:43:49.114209+0000 mgr.y (mgr.34101) 896 : cluster 0 pgmap v1472: 105 pgs: 105 active+clean; 75 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 44 op/s 2024-03-07T03:43:51.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:50 smithi148 bash[82222]: cluster 2024-03-07T03:43:49.114209+0000 mgr.y (mgr.34101) 896 : cluster 0 pgmap v1472: 105 pgs: 105 active+clean; 75 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 44 op/s 2024-03-07T03:43:51.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:50 smithi195 bash[66542]: cluster 2024-03-07T03:43:49.114209+0000 mgr.y (mgr.34101) 896 : cluster 0 pgmap v1472: 105 pgs: 105 active+clean; 75 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 6.1 MiB/s wr, 44 op/s 2024-03-07T03:43:52.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:43:52 smithi195 bash[59134]: ts=2024-03-07T03:43:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:43:53.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:52 smithi148 bash[79819]: cluster 2024-03-07T03:43:51.114794+0000 mgr.y (mgr.34101) 897 : cluster 0 pgmap v1473: 105 pgs: 105 active+clean; 79 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.8 MiB/s wr, 43 op/s 2024-03-07T03:43:53.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:52 smithi148 bash[82222]: cluster 2024-03-07T03:43:51.114794+0000 mgr.y (mgr.34101) 897 : cluster 0 pgmap v1473: 105 pgs: 105 active+clean; 79 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.8 MiB/s wr, 43 op/s 2024-03-07T03:43:53.067 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:52 smithi195 bash[66542]: cluster 2024-03-07T03:43:51.114794+0000 mgr.y (mgr.34101) 897 : cluster 0 pgmap v1473: 105 pgs: 105 active+clean; 79 MiB data, 860 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 5.8 MiB/s wr, 43 op/s 2024-03-07T03:43:55.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:54 smithi148 bash[79819]: cluster 2024-03-07T03:43:53.115323+0000 mgr.y (mgr.34101) 898 : cluster 0 pgmap v1474: 105 pgs: 105 active+clean; 83 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:43:55.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:54 smithi148 bash[82222]: cluster 2024-03-07T03:43:53.115323+0000 mgr.y (mgr.34101) 898 : cluster 0 pgmap v1474: 105 pgs: 105 active+clean; 83 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:43:55.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:54 smithi195 bash[66542]: cluster 2024-03-07T03:43:53.115323+0000 mgr.y (mgr.34101) 898 : cluster 0 pgmap v1474: 105 pgs: 105 active+clean; 83 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:43:57.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:56 smithi148 bash[79819]: cluster 2024-03-07T03:43:55.116587+0000 mgr.y (mgr.34101) 899 : cluster 0 pgmap v1475: 105 pgs: 105 active+clean; 87 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.1 MiB/s wr, 29 op/s 2024-03-07T03:43:57.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:56 smithi148 bash[79819]: cluster 2024-03-07T03:43:55.784747+0000 mon.a (mon.0) 1713 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2024-03-07T03:43:57.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:56 smithi148 bash[82222]: cluster 2024-03-07T03:43:55.116587+0000 mgr.y (mgr.34101) 899 : cluster 0 pgmap v1475: 105 pgs: 105 active+clean; 87 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.1 MiB/s wr, 29 op/s 2024-03-07T03:43:57.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:56 smithi148 bash[82222]: cluster 2024-03-07T03:43:55.784747+0000 mon.a (mon.0) 1713 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2024-03-07T03:43:57.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:56 smithi195 bash[66542]: cluster 2024-03-07T03:43:55.116587+0000 mgr.y (mgr.34101) 899 : cluster 0 pgmap v1475: 105 pgs: 105 active+clean; 87 MiB data, 868 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.1 MiB/s wr, 29 op/s 2024-03-07T03:43:57.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:56 smithi195 bash[66542]: cluster 2024-03-07T03:43:55.784747+0000 mon.a (mon.0) 1713 : cluster 0 osdmap e958: 8 total, 8 up, 8 in 2024-03-07T03:43:57.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:43:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:43:57] "GET /metrics HTTP/1.1" 200 36735 "" "Prometheus/2.43.0" 2024-03-07T03:43:58.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:57 smithi148 bash[79819]: audit 2024-03-07T03:43:57.343276+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:58.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:57 smithi148 bash[82222]: audit 2024-03-07T03:43:57.343276+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:58.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:57 smithi195 bash[66542]: audit 2024-03-07T03:43:57.343276+0000 mon.a (mon.0) 1714 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:43:59.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:43:58 smithi148 bash[79819]: cluster 2024-03-07T03:43:57.117220+0000 mgr.y (mgr.34101) 900 : cluster 0 pgmap v1477: 105 pgs: 105 active+clean; 99 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 40 op/s 2024-03-07T03:43:59.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:43:58 smithi148 bash[82222]: cluster 2024-03-07T03:43:57.117220+0000 mgr.y (mgr.34101) 900 : cluster 0 pgmap v1477: 105 pgs: 105 active+clean; 99 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 40 op/s 2024-03-07T03:43:59.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:43:58 smithi195 bash[66542]: cluster 2024-03-07T03:43:57.117220+0000 mgr.y (mgr.34101) 900 : cluster 0 pgmap v1477: 105 pgs: 105 active+clean; 99 MiB data, 888 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 40 op/s 2024-03-07T03:44:01.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:00 smithi148 bash[79819]: cluster 2024-03-07T03:43:59.117867+0000 mgr.y (mgr.34101) 901 : cluster 0 pgmap v1478: 105 pgs: 105 active+clean; 124 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.1 MiB/s wr, 47 op/s 2024-03-07T03:44:01.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:00 smithi148 bash[82222]: cluster 2024-03-07T03:43:59.117867+0000 mgr.y (mgr.34101) 901 : cluster 0 pgmap v1478: 105 pgs: 105 active+clean; 124 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.1 MiB/s wr, 47 op/s 2024-03-07T03:44:01.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:00 smithi195 bash[66542]: cluster 2024-03-07T03:43:59.117867+0000 mgr.y (mgr.34101) 901 : cluster 0 pgmap v1478: 105 pgs: 105 active+clean; 124 MiB data, 971 MiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 6.1 MiB/s wr, 47 op/s 2024-03-07T03:44:02.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:44:02 smithi195 bash[59134]: ts=2024-03-07T03:44:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:03.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:02 smithi148 bash[79819]: cluster 2024-03-07T03:44:01.118473+0000 mgr.y (mgr.34101) 902 : cluster 0 pgmap v1479: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-03-07T03:44:03.045 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:02 smithi148 bash[82222]: cluster 2024-03-07T03:44:01.118473+0000 mgr.y (mgr.34101) 902 : cluster 0 pgmap v1479: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-03-07T03:44:03.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:02 smithi195 bash[66542]: cluster 2024-03-07T03:44:01.118473+0000 mgr.y (mgr.34101) 902 : cluster 0 pgmap v1479: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-03-07T03:44:05.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:04 smithi148 bash[79819]: cluster 2024-03-07T03:44:03.119032+0000 mgr.y (mgr.34101) 903 : cluster 0 pgmap v1480: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:44:05.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:04 smithi148 bash[82222]: cluster 2024-03-07T03:44:03.119032+0000 mgr.y (mgr.34101) 903 : cluster 0 pgmap v1480: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:44:05.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:04 smithi195 bash[66542]: cluster 2024-03-07T03:44:03.119032+0000 mgr.y (mgr.34101) 903 : cluster 0 pgmap v1480: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:44:07.045 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:06 smithi148 bash[79819]: cluster 2024-03-07T03:44:05.119825+0000 mgr.y (mgr.34101) 904 : cluster 0 pgmap v1481: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-03-07T03:44:07.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:06 smithi148 bash[82222]: cluster 2024-03-07T03:44:05.119825+0000 mgr.y (mgr.34101) 904 : cluster 0 pgmap v1481: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-03-07T03:44:07.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:06 smithi195 bash[66542]: cluster 2024-03-07T03:44:05.119825+0000 mgr.y (mgr.34101) 904 : cluster 0 pgmap v1481: 105 pgs: 105 active+clean; 124 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.7 MiB/s wr, 35 op/s 2024-03-07T03:44:07.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:44:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:44:07] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2024-03-07T03:44:09.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:08 smithi195 bash[66542]: cluster 2024-03-07T03:44:07.120466+0000 mgr.y (mgr.34101) 905 : cluster 0 pgmap v1482: 105 pgs: 105 active+clean; 132 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.0 MiB/s wr, 34 op/s 2024-03-07T03:44:09.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:08 smithi148 bash[79819]: cluster 2024-03-07T03:44:07.120466+0000 mgr.y (mgr.34101) 905 : cluster 0 pgmap v1482: 105 pgs: 105 active+clean; 132 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.0 MiB/s wr, 34 op/s 2024-03-07T03:44:09.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:08 smithi148 bash[82222]: cluster 2024-03-07T03:44:07.120466+0000 mgr.y (mgr.34101) 905 : cluster 0 pgmap v1482: 105 pgs: 105 active+clean; 132 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 4.0 MiB/s wr, 34 op/s 2024-03-07T03:44:11.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:10 smithi195 bash[66542]: cluster 2024-03-07T03:44:09.121210+0000 mgr.y (mgr.34101) 906 : cluster 0 pgmap v1483: 105 pgs: 105 active+clean; 136 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-03-07T03:44:11.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:10 smithi148 bash[79819]: cluster 2024-03-07T03:44:09.121210+0000 mgr.y (mgr.34101) 906 : cluster 0 pgmap v1483: 105 pgs: 105 active+clean; 136 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-03-07T03:44:11.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:10 smithi148 bash[82222]: cluster 2024-03-07T03:44:09.121210+0000 mgr.y (mgr.34101) 906 : cluster 0 pgmap v1483: 105 pgs: 105 active+clean; 136 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 63 KiB/s rd, 3.1 MiB/s wr, 21 op/s 2024-03-07T03:44:11.971 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/6.Grow (23401 ms) 2024-03-07T03:44:11.971 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/6.Shrink 2024-03-07T03:44:11.971 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:44:12.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:44:12 smithi195 bash[59134]: ts=2024-03-07T03:44:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:13.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:12 smithi195 bash[66542]: cluster 2024-03-07T03:44:11.121824+0000 mgr.y (mgr.34101) 907 : cluster 0 pgmap v1484: 105 pgs: 105 active+clean; 157 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.1 MiB/s wr, 22 op/s 2024-03-07T03:44:13.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:12 smithi195 bash[66542]: audit 2024-03-07T03:44:12.344058+0000 mon.a (mon.0) 1715 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:13.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:12 smithi148 bash[79819]: cluster 2024-03-07T03:44:11.121824+0000 mgr.y (mgr.34101) 907 : cluster 0 pgmap v1484: 105 pgs: 105 active+clean; 157 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.1 MiB/s wr, 22 op/s 2024-03-07T03:44:13.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:12 smithi148 bash[79819]: audit 2024-03-07T03:44:12.344058+0000 mon.a (mon.0) 1715 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:13.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:12 smithi148 bash[82222]: cluster 2024-03-07T03:44:11.121824+0000 mgr.y (mgr.34101) 907 : cluster 0 pgmap v1484: 105 pgs: 105 active+clean; 157 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 3.1 MiB/s wr, 22 op/s 2024-03-07T03:44:13.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:12 smithi148 bash[82222]: audit 2024-03-07T03:44:12.344058+0000 mon.a (mon.0) 1715 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:15.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:14 smithi195 bash[66542]: cluster 2024-03-07T03:44:13.122516+0000 mgr.y (mgr.34101) 908 : cluster 0 pgmap v1485: 105 pgs: 105 active+clean; 161 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.1 MiB/s wr, 18 op/s 2024-03-07T03:44:15.295 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:14 smithi148 bash[79819]: cluster 2024-03-07T03:44:13.122516+0000 mgr.y (mgr.34101) 908 : cluster 0 pgmap v1485: 105 pgs: 105 active+clean; 161 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.1 MiB/s wr, 18 op/s 2024-03-07T03:44:15.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:14 smithi148 bash[82222]: cluster 2024-03-07T03:44:13.122516+0000 mgr.y (mgr.34101) 908 : cluster 0 pgmap v1485: 105 pgs: 105 active+clean; 161 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.1 MiB/s wr, 18 op/s 2024-03-07T03:44:17.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:16 smithi195 bash[66542]: cluster 2024-03-07T03:44:15.123297+0000 mgr.y (mgr.34101) 909 : cluster 0 pgmap v1486: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:44:17.173 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:16 smithi148 bash[82222]: cluster 2024-03-07T03:44:15.123297+0000 mgr.y (mgr.34101) 909 : cluster 0 pgmap v1486: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:44:17.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:16 smithi148 bash[79819]: cluster 2024-03-07T03:44:15.123297+0000 mgr.y (mgr.34101) 909 : cluster 0 pgmap v1486: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:44:17.545 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:44:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:44:17] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2024-03-07T03:44:19.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[79819]: cluster 2024-03-07T03:44:17.123927+0000 mgr.y (mgr.34101) 910 : cluster 0 pgmap v1487: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:44:19.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[79819]: audit 2024-03-07T03:44:17.844491+0000 mon.a (mon.0) 1716 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.1", "id": [0, 7]}]: dispatch 2024-03-07T03:44:19.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[79819]: audit 2024-03-07T03:44:17.844688+0000 mon.a (mon.0) 1717 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.9", "id": [0, 4]}]: dispatch 2024-03-07T03:44:19.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[79819]: audit 2024-03-07T03:44:17.844830+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.12", "id": [0, 2]}]: dispatch 2024-03-07T03:44:19.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[79819]: audit 2024-03-07T03:44:17.844966+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]: dispatch 2024-03-07T03:44:19.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[82222]: cluster 2024-03-07T03:44:17.123927+0000 mgr.y (mgr.34101) 910 : cluster 0 pgmap v1487: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:44:19.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[82222]: audit 2024-03-07T03:44:17.844491+0000 mon.a (mon.0) 1716 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.1", "id": [0, 7]}]: dispatch 2024-03-07T03:44:19.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[82222]: audit 2024-03-07T03:44:17.844688+0000 mon.a (mon.0) 1717 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.9", "id": [0, 4]}]: dispatch 2024-03-07T03:44:19.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[82222]: audit 2024-03-07T03:44:17.844830+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.12", "id": [0, 2]}]: dispatch 2024-03-07T03:44:19.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:18 smithi148 bash[82222]: audit 2024-03-07T03:44:17.844966+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]: dispatch 2024-03-07T03:44:19.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:18 smithi195 bash[66542]: cluster 2024-03-07T03:44:17.123927+0000 mgr.y (mgr.34101) 910 : cluster 0 pgmap v1487: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-03-07T03:44:19.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:18 smithi195 bash[66542]: audit 2024-03-07T03:44:17.844491+0000 mon.a (mon.0) 1716 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.1", "id": [0, 7]}]: dispatch 2024-03-07T03:44:19.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:18 smithi195 bash[66542]: audit 2024-03-07T03:44:17.844688+0000 mon.a (mon.0) 1717 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.9", "id": [0, 4]}]: dispatch 2024-03-07T03:44:19.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:18 smithi195 bash[66542]: audit 2024-03-07T03:44:17.844830+0000 mon.a (mon.0) 1718 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.12", "id": [0, 2]}]: dispatch 2024-03-07T03:44:19.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:18 smithi195 bash[66542]: audit 2024-03-07T03:44:17.844966+0000 mon.a (mon.0) 1719 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]: dispatch 2024-03-07T03:44:20.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[79819]: audit 2024-03-07T03:44:18.830670+0000 mon.a (mon.0) 1720 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.1", "id": [0, 7]}]': finished 2024-03-07T03:44:20.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[79819]: audit 2024-03-07T03:44:18.830754+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.9", "id": [0, 4]}]': finished 2024-03-07T03:44:20.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[79819]: audit 2024-03-07T03:44:18.830816+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.12", "id": [0, 2]}]': finished 2024-03-07T03:44:20.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[79819]: audit 2024-03-07T03:44:18.830877+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]': finished 2024-03-07T03:44:20.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[79819]: cluster 2024-03-07T03:44:18.836568+0000 mon.a (mon.0) 1724 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2024-03-07T03:44:20.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[79819]: cluster 2024-03-07T03:44:19.124690+0000 mgr.y (mgr.34101) 911 : cluster 0 pgmap v1489: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 28 op/s 2024-03-07T03:44:20.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[82222]: audit 2024-03-07T03:44:18.830670+0000 mon.a (mon.0) 1720 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.1", "id": [0, 7]}]': finished 2024-03-07T03:44:20.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[82222]: audit 2024-03-07T03:44:18.830754+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.9", "id": [0, 4]}]': finished 2024-03-07T03:44:20.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[82222]: audit 2024-03-07T03:44:18.830816+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.12", "id": [0, 2]}]': finished 2024-03-07T03:44:20.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[82222]: audit 2024-03-07T03:44:18.830877+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]': finished 2024-03-07T03:44:20.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[82222]: cluster 2024-03-07T03:44:18.836568+0000 mon.a (mon.0) 1724 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2024-03-07T03:44:20.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:19 smithi148 bash[82222]: cluster 2024-03-07T03:44:19.124690+0000 mgr.y (mgr.34101) 911 : cluster 0 pgmap v1489: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 28 op/s 2024-03-07T03:44:20.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:19 smithi195 bash[66542]: audit 2024-03-07T03:44:18.830670+0000 mon.a (mon.0) 1720 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.1", "id": [0, 7]}]': finished 2024-03-07T03:44:20.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:19 smithi195 bash[66542]: audit 2024-03-07T03:44:18.830754+0000 mon.a (mon.0) 1721 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.9", "id": [0, 4]}]': finished 2024-03-07T03:44:20.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:19 smithi195 bash[66542]: audit 2024-03-07T03:44:18.830816+0000 mon.a (mon.0) 1722 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.12", "id": [0, 2]}]': finished 2024-03-07T03:44:20.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:19 smithi195 bash[66542]: audit 2024-03-07T03:44:18.830877+0000 mon.a (mon.0) 1723 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]': finished 2024-03-07T03:44:20.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:19 smithi195 bash[66542]: cluster 2024-03-07T03:44:18.836568+0000 mon.a (mon.0) 1724 : cluster 0 osdmap e959: 8 total, 8 up, 8 in 2024-03-07T03:44:20.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:19 smithi195 bash[66542]: cluster 2024-03-07T03:44:19.124690+0000 mgr.y (mgr.34101) 911 : cluster 0 pgmap v1489: 105 pgs: 105 active+clean; 165 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.5 MiB/s wr, 28 op/s 2024-03-07T03:44:21.295 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:20 smithi148 bash[82222]: cluster 2024-03-07T03:44:19.860022+0000 mon.a (mon.0) 1725 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2024-03-07T03:44:21.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:20 smithi148 bash[79819]: cluster 2024-03-07T03:44:19.860022+0000 mon.a (mon.0) 1725 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2024-03-07T03:44:21.318 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:20 smithi195 bash[66542]: cluster 2024-03-07T03:44:19.860022+0000 mon.a (mon.0) 1725 : cluster 0 osdmap e960: 8 total, 8 up, 8 in 2024-03-07T03:44:22.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:21 smithi148 bash[79819]: cluster 2024-03-07T03:44:20.869327+0000 mon.a (mon.0) 1726 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2024-03-07T03:44:22.296 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:21 smithi148 bash[79819]: cluster 2024-03-07T03:44:21.125209+0000 mgr.y (mgr.34101) 912 : cluster 0 pgmap v1492: 105 pgs: 105 active+clean; 177 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.0 MiB/s wr, 7 op/s 2024-03-07T03:44:22.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:21 smithi148 bash[82222]: cluster 2024-03-07T03:44:20.869327+0000 mon.a (mon.0) 1726 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2024-03-07T03:44:22.296 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:21 smithi148 bash[82222]: cluster 2024-03-07T03:44:21.125209+0000 mgr.y (mgr.34101) 912 : cluster 0 pgmap v1492: 105 pgs: 105 active+clean; 177 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.0 MiB/s wr, 7 op/s 2024-03-07T03:44:22.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:21 smithi195 bash[66542]: cluster 2024-03-07T03:44:20.869327+0000 mon.a (mon.0) 1726 : cluster 0 osdmap e961: 8 total, 8 up, 8 in 2024-03-07T03:44:22.309 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:21 smithi195 bash[66542]: cluster 2024-03-07T03:44:21.125209+0000 mgr.y (mgr.34101) 912 : cluster 0 pgmap v1492: 105 pgs: 105 active+clean; 177 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.0 MiB/s wr, 7 op/s 2024-03-07T03:44:22.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:44:22 smithi195 bash[59134]: ts=2024-03-07T03:44:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:24.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:24 smithi148 bash[79819]: cluster 2024-03-07T03:44:23.125856+0000 mgr.y (mgr.34101) 913 : cluster 0 pgmap v1493: 105 pgs: 105 active+clean; 185 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.3 MiB/s wr, 16 op/s 2024-03-07T03:44:24.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:24 smithi148 bash[82222]: cluster 2024-03-07T03:44:23.125856+0000 mgr.y (mgr.34101) 913 : cluster 0 pgmap v1493: 105 pgs: 105 active+clean; 185 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.3 MiB/s wr, 16 op/s 2024-03-07T03:44:24.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:24 smithi195 bash[66542]: cluster 2024-03-07T03:44:23.125856+0000 mgr.y (mgr.34101) 913 : cluster 0 pgmap v1493: 105 pgs: 105 active+clean; 185 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.8 KiB/s rd, 3.3 MiB/s wr, 16 op/s 2024-03-07T03:44:26.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:26 smithi148 bash[79819]: cluster 2024-03-07T03:44:25.126648+0000 mgr.y (mgr.34101) 914 : cluster 0 pgmap v1494: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.5 MiB/s wr, 56 op/s; 1.9 MiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:44:26.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:26 smithi148 bash[82222]: cluster 2024-03-07T03:44:25.126648+0000 mgr.y (mgr.34101) 914 : cluster 0 pgmap v1494: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.5 MiB/s wr, 56 op/s; 1.9 MiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:44:26.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:26 smithi195 bash[66542]: cluster 2024-03-07T03:44:25.126648+0000 mgr.y (mgr.34101) 914 : cluster 0 pgmap v1494: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 6.5 MiB/s wr, 56 op/s; 1.9 MiB/s, 1 keys/s, 0 objects/s recovering 2024-03-07T03:44:27.546 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:44:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:44:27] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2024-03-07T03:44:28.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:28 smithi148 bash[79819]: cluster 2024-03-07T03:44:27.127287+0000 mgr.y (mgr.34101) 915 : cluster 0 pgmap v1495: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s; 1.5 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:28.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:28 smithi148 bash[79819]: audit 2024-03-07T03:44:27.343792+0000 mon.a (mon.0) 1727 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:28.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:28 smithi148 bash[82222]: cluster 2024-03-07T03:44:27.127287+0000 mgr.y (mgr.34101) 915 : cluster 0 pgmap v1495: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s; 1.5 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:28.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:28 smithi148 bash[82222]: audit 2024-03-07T03:44:27.343792+0000 mon.a (mon.0) 1727 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:28.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:28 smithi195 bash[66542]: cluster 2024-03-07T03:44:27.127287+0000 mgr.y (mgr.34101) 915 : cluster 0 pgmap v1495: 105 pgs: 105 active+clean; 206 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s; 1.5 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:28.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:28 smithi195 bash[66542]: audit 2024-03-07T03:44:27.343792+0000 mon.a (mon.0) 1727 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:30.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:30 smithi148 bash[79819]: cluster 2024-03-07T03:44:29.128051+0000 mgr.y (mgr.34101) 916 : cluster 0 pgmap v1496: 105 pgs: 105 active+clean; 218 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 5.7 MiB/s wr, 49 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:30.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:30 smithi148 bash[82222]: cluster 2024-03-07T03:44:29.128051+0000 mgr.y (mgr.34101) 916 : cluster 0 pgmap v1496: 105 pgs: 105 active+clean; 218 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 5.7 MiB/s wr, 49 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:30.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:30 smithi195 bash[66542]: cluster 2024-03-07T03:44:29.128051+0000 mgr.y (mgr.34101) 916 : cluster 0 pgmap v1496: 105 pgs: 105 active+clean; 218 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 5.7 MiB/s wr, 49 op/s; 1.3 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:32.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:32 smithi148 bash[79819]: cluster 2024-03-07T03:44:31.128649+0000 mgr.y (mgr.34101) 917 : cluster 0 pgmap v1497: 105 pgs: 105 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.4 MiB/s wr, 44 op/s; 1.2 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:32.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:32 smithi148 bash[82222]: cluster 2024-03-07T03:44:31.128649+0000 mgr.y (mgr.34101) 917 : cluster 0 pgmap v1497: 105 pgs: 105 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.4 MiB/s wr, 44 op/s; 1.2 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:32.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:32 smithi195 bash[66542]: cluster 2024-03-07T03:44:31.128649+0000 mgr.y (mgr.34101) 917 : cluster 0 pgmap v1497: 105 pgs: 105 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 4.4 MiB/s wr, 44 op/s; 1.2 MiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:32.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:44:32 smithi195 bash[59134]: ts=2024-03-07T03:44:32.321Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:34.303 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/6.Shrink (22332 ms) 2024-03-07T03:44:34.303 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/6.ShrinkToOne 2024-03-07T03:44:34.303 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:44:34.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:34 smithi148 bash[82222]: cluster 2024-03-07T03:44:33.129223+0000 mgr.y (mgr.34101) 918 : cluster 0 pgmap v1498: 105 pgs: 105 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 3.8 MiB/s wr, 38 op/s; 1024 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:34.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:34 smithi148 bash[79819]: cluster 2024-03-07T03:44:33.129223+0000 mgr.y (mgr.34101) 918 : cluster 0 pgmap v1498: 105 pgs: 105 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 3.8 MiB/s wr, 38 op/s; 1024 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:34.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:34 smithi195 bash[66542]: cluster 2024-03-07T03:44:33.129223+0000 mgr.y (mgr.34101) 918 : cluster 0 pgmap v1498: 105 pgs: 105 active+clean; 222 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 3.8 MiB/s wr, 38 op/s; 1024 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:36.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:36 smithi148 bash[79819]: cluster 2024-03-07T03:44:35.130311+0000 mgr.y (mgr.34101) 919 : cluster 0 pgmap v1499: 105 pgs: 105 active+clean; 239 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 5.5 MiB/s wr, 47 op/s; 1024 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:36.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:36 smithi148 bash[82222]: cluster 2024-03-07T03:44:35.130311+0000 mgr.y (mgr.34101) 919 : cluster 0 pgmap v1499: 105 pgs: 105 active+clean; 239 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 5.5 MiB/s wr, 47 op/s; 1024 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:36.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:36 smithi195 bash[66542]: cluster 2024-03-07T03:44:35.130311+0000 mgr.y (mgr.34101) 919 : cluster 0 pgmap v1499: 105 pgs: 105 active+clean; 239 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 5.5 MiB/s wr, 47 op/s; 1024 KiB/s, 0 keys/s, 0 objects/s recovering 2024-03-07T03:44:37.546 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:44:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:44:37] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.43.0" 2024-03-07T03:44:38.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:38 smithi148 bash[79819]: cluster 2024-03-07T03:44:37.130978+0000 mgr.y (mgr.34101) 920 : cluster 0 pgmap v1500: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:44:38.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:38 smithi148 bash[79819]: audit 2024-03-07T03:44:38.157495+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:44:38.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:38 smithi148 bash[82222]: cluster 2024-03-07T03:44:37.130978+0000 mgr.y (mgr.34101) 920 : cluster 0 pgmap v1500: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:44:38.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:38 smithi148 bash[82222]: audit 2024-03-07T03:44:38.157495+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:44:38.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:38 smithi195 bash[66542]: cluster 2024-03-07T03:44:37.130978+0000 mgr.y (mgr.34101) 920 : cluster 0 pgmap v1500: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:44:38.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:38 smithi195 bash[66542]: audit 2024-03-07T03:44:38.157495+0000 mon.a (mon.0) 1728 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:44:40.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:40 smithi148 bash[79819]: cluster 2024-03-07T03:44:39.131748+0000 mgr.y (mgr.34101) 921 : cluster 0 pgmap v1501: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:44:40.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:40 smithi148 bash[82222]: cluster 2024-03-07T03:44:39.131748+0000 mgr.y (mgr.34101) 921 : cluster 0 pgmap v1501: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:44:40.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:40 smithi195 bash[66542]: cluster 2024-03-07T03:44:39.131748+0000 mgr.y (mgr.34101) 921 : cluster 0 pgmap v1501: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-03-07T03:44:42.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:42 smithi148 bash[79819]: cluster 2024-03-07T03:44:41.132386+0000 mgr.y (mgr.34101) 922 : cluster 0 pgmap v1502: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.8 MiB/s wr, 22 op/s 2024-03-07T03:44:42.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:42 smithi148 bash[82222]: cluster 2024-03-07T03:44:41.132386+0000 mgr.y (mgr.34101) 922 : cluster 0 pgmap v1502: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.8 MiB/s wr, 22 op/s 2024-03-07T03:44:42.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:42 smithi195 bash[66542]: cluster 2024-03-07T03:44:41.132386+0000 mgr.y (mgr.34101) 922 : cluster 0 pgmap v1502: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.8 MiB/s wr, 22 op/s 2024-03-07T03:44:42.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:44:42 smithi195 bash[59134]: ts=2024-03-07T03:44:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:43.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:43 smithi148 bash[79819]: cluster 2024-03-07T03:44:42.228387+0000 mon.a (mon.0) 1729 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2024-03-07T03:44:43.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:43 smithi148 bash[79819]: audit 2024-03-07T03:44:42.345016+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:43.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:43 smithi148 bash[82222]: cluster 2024-03-07T03:44:42.228387+0000 mon.a (mon.0) 1729 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2024-03-07T03:44:43.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:43 smithi148 bash[82222]: audit 2024-03-07T03:44:42.345016+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:43.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:43 smithi195 bash[66542]: cluster 2024-03-07T03:44:42.228387+0000 mon.a (mon.0) 1729 : cluster 0 osdmap e962: 8 total, 8 up, 8 in 2024-03-07T03:44:43.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:43 smithi195 bash[66542]: audit 2024-03-07T03:44:42.345016+0000 mon.a (mon.0) 1730 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:44.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[79819]: cluster 2024-03-07T03:44:43.132806+0000 mgr.y (mgr.34101) 923 : cluster 0 pgmap v1504: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:44:44.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[79819]: audit 2024-03-07T03:44:43.953766+0000 mon.a (mon.0) 1731 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[79819]: audit 2024-03-07T03:44:43.959906+0000 mon.a (mon.0) 1732 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[79819]: audit 2024-03-07T03:44:44.156781+0000 mon.a (mon.0) 1733 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[79819]: audit 2024-03-07T03:44:44.166192+0000 mon.a (mon.0) 1734 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[82222]: cluster 2024-03-07T03:44:43.132806+0000 mgr.y (mgr.34101) 923 : cluster 0 pgmap v1504: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:44:44.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[82222]: audit 2024-03-07T03:44:43.953766+0000 mon.a (mon.0) 1731 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[82222]: audit 2024-03-07T03:44:43.959906+0000 mon.a (mon.0) 1732 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[82222]: audit 2024-03-07T03:44:44.156781+0000 mon.a (mon.0) 1733 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:44 smithi148 bash[82222]: audit 2024-03-07T03:44:44.166192+0000 mon.a (mon.0) 1734 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:44 smithi195 bash[66542]: cluster 2024-03-07T03:44:43.132806+0000 mgr.y (mgr.34101) 923 : cluster 0 pgmap v1504: 105 pgs: 105 active+clean; 247 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:44:44.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:44 smithi195 bash[66542]: audit 2024-03-07T03:44:43.953766+0000 mon.a (mon.0) 1731 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:44 smithi195 bash[66542]: audit 2024-03-07T03:44:43.959906+0000 mon.a (mon.0) 1732 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:44 smithi195 bash[66542]: audit 2024-03-07T03:44:44.156781+0000 mon.a (mon.0) 1733 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:44.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:44 smithi195 bash[66542]: audit 2024-03-07T03:44:44.166192+0000 mon.a (mon.0) 1734 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:45.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:45 smithi148 bash[79819]: audit 2024-03-07T03:44:44.694843+0000 mon.a (mon.0) 1735 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:44:45.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:45 smithi148 bash[79819]: audit 2024-03-07T03:44:44.695635+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:44:45.546 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:45 smithi148 bash[79819]: audit 2024-03-07T03:44:44.702931+0000 mon.a (mon.0) 1737 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:45.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:45 smithi148 bash[82222]: audit 2024-03-07T03:44:44.694843+0000 mon.a (mon.0) 1735 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:44:45.546 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:45 smithi148 bash[82222]: audit 2024-03-07T03:44:44.695635+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:44:45.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:45 smithi148 bash[82222]: audit 2024-03-07T03:44:44.702931+0000 mon.a (mon.0) 1737 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:45.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:45 smithi195 bash[66542]: audit 2024-03-07T03:44:44.694843+0000 mon.a (mon.0) 1735 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:44:45.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:45 smithi195 bash[66542]: audit 2024-03-07T03:44:44.695635+0000 mon.a (mon.0) 1736 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:44:45.568 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:45 smithi195 bash[66542]: audit 2024-03-07T03:44:44.702931+0000 mon.a (mon.0) 1737 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:44:47.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:46 smithi148 bash[79819]: cluster 2024-03-07T03:44:45.133588+0000 mgr.y (mgr.34101) 924 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-03-07T03:44:47.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:46 smithi148 bash[82222]: cluster 2024-03-07T03:44:45.133588+0000 mgr.y (mgr.34101) 924 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-03-07T03:44:47.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:46 smithi195 bash[66542]: cluster 2024-03-07T03:44:45.133588+0000 mgr.y (mgr.34101) 924 : cluster 0 pgmap v1505: 105 pgs: 105 active+clean; 280 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-03-07T03:44:47.546 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:44:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:44:47] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-03-07T03:44:49.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:48 smithi148 bash[79819]: cluster 2024-03-07T03:44:47.134199+0000 mgr.y (mgr.34101) 925 : cluster 0 pgmap v1506: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 950 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:49.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:48 smithi148 bash[82222]: cluster 2024-03-07T03:44:47.134199+0000 mgr.y (mgr.34101) 925 : cluster 0 pgmap v1506: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 950 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:49.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:48 smithi195 bash[66542]: cluster 2024-03-07T03:44:47.134199+0000 mgr.y (mgr.34101) 925 : cluster 0 pgmap v1506: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 950 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:51.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:50 smithi148 bash[79819]: cluster 2024-03-07T03:44:49.134972+0000 mgr.y (mgr.34101) 926 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:51.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:50 smithi148 bash[82222]: cluster 2024-03-07T03:44:49.134972+0000 mgr.y (mgr.34101) 926 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:51.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:50 smithi195 bash[66542]: cluster 2024-03-07T03:44:49.134972+0000 mgr.y (mgr.34101) 926 : cluster 0 pgmap v1507: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:44:52.568 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:44:52 smithi195 bash[59134]: ts=2024-03-07T03:44:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:44:53.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:52 smithi148 bash[79819]: cluster 2024-03-07T03:44:51.135563+0000 mgr.y (mgr.34101) 927 : cluster 0 pgmap v1508: 105 pgs: 105 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:44:53.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:52 smithi148 bash[82222]: cluster 2024-03-07T03:44:51.135563+0000 mgr.y (mgr.34101) 927 : cluster 0 pgmap v1508: 105 pgs: 105 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:44:53.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:52 smithi195 bash[66542]: cluster 2024-03-07T03:44:51.135563+0000 mgr.y (mgr.34101) 927 : cluster 0 pgmap v1508: 105 pgs: 105 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 4.5 MiB/s wr, 42 op/s 2024-03-07T03:44:55.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:54 smithi148 bash[79819]: cluster 2024-03-07T03:44:53.136333+0000 mgr.y (mgr.34101) 928 : cluster 0 pgmap v1509: 105 pgs: 105 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:55.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:54 smithi148 bash[82222]: cluster 2024-03-07T03:44:53.136333+0000 mgr.y (mgr.34101) 928 : cluster 0 pgmap v1509: 105 pgs: 105 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:55.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:54 smithi195 bash[66542]: cluster 2024-03-07T03:44:53.136333+0000 mgr.y (mgr.34101) 928 : cluster 0 pgmap v1509: 105 pgs: 105 active+clean; 292 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-03-07T03:44:55.243 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/6.ShrinkToOne (20939 ms) 2024-03-07T03:44:55.243 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/6.ShrinkToOneAfterSnapshot 2024-03-07T03:44:55.243 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:44:57.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:56 smithi148 bash[79819]: cluster 2024-03-07T03:44:55.137568+0000 mgr.y (mgr.34101) 929 : cluster 0 pgmap v1510: 105 pgs: 105 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.4 MiB/s wr, 41 op/s 2024-03-07T03:44:57.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:56 smithi148 bash[82222]: cluster 2024-03-07T03:44:55.137568+0000 mgr.y (mgr.34101) 929 : cluster 0 pgmap v1510: 105 pgs: 105 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.4 MiB/s wr, 41 op/s 2024-03-07T03:44:57.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:56 smithi195 bash[66542]: cluster 2024-03-07T03:44:55.137568+0000 mgr.y (mgr.34101) 929 : cluster 0 pgmap v1510: 105 pgs: 105 active+clean; 304 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.4 MiB/s wr, 41 op/s 2024-03-07T03:44:57.546 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:44:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:44:57] "GET /metrics HTTP/1.1" 200 36742 "" "Prometheus/2.43.0" 2024-03-07T03:44:58.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:57 smithi148 bash[79819]: audit 2024-03-07T03:44:57.344810+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:58.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:57 smithi148 bash[82222]: audit 2024-03-07T03:44:57.344810+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:58.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:57 smithi195 bash[66542]: audit 2024-03-07T03:44:57.344810+0000 mon.a (mon.0) 1738 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:44:59.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:44:58 smithi148 bash[79819]: cluster 2024-03-07T03:44:57.138295+0000 mgr.y (mgr.34101) 930 : cluster 0 pgmap v1511: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-03-07T03:44:59.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:44:58 smithi148 bash[82222]: cluster 2024-03-07T03:44:57.138295+0000 mgr.y (mgr.34101) 930 : cluster 0 pgmap v1511: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-03-07T03:44:59.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:44:58 smithi195 bash[66542]: cluster 2024-03-07T03:44:57.138295+0000 mgr.y (mgr.34101) 930 : cluster 0 pgmap v1511: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.7 MiB/s wr, 23 op/s 2024-03-07T03:45:01.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:00 smithi148 bash[79819]: cluster 2024-03-07T03:44:59.139052+0000 mgr.y (mgr.34101) 931 : cluster 0 pgmap v1512: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-03-07T03:45:01.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:00 smithi148 bash[82222]: cluster 2024-03-07T03:44:59.139052+0000 mgr.y (mgr.34101) 931 : cluster 0 pgmap v1512: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-03-07T03:45:01.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:00 smithi195 bash[66542]: cluster 2024-03-07T03:44:59.139052+0000 mgr.y (mgr.34101) 931 : cluster 0 pgmap v1512: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-03-07T03:45:02.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:45:02 smithi195 bash[59134]: ts=2024-03-07T03:45:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:03.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:02 smithi148 bash[79819]: cluster 2024-03-07T03:45:01.139648+0000 mgr.y (mgr.34101) 932 : cluster 0 pgmap v1513: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:45:03.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:02 smithi148 bash[82222]: cluster 2024-03-07T03:45:01.139648+0000 mgr.y (mgr.34101) 932 : cluster 0 pgmap v1513: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:45:03.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:02 smithi195 bash[66542]: cluster 2024-03-07T03:45:01.139648+0000 mgr.y (mgr.34101) 932 : cluster 0 pgmap v1513: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-03-07T03:45:05.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:04 smithi148 bash[79819]: cluster 2024-03-07T03:45:03.140218+0000 mgr.y (mgr.34101) 933 : cluster 0 pgmap v1514: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2024-03-07T03:45:05.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:04 smithi148 bash[82222]: cluster 2024-03-07T03:45:03.140218+0000 mgr.y (mgr.34101) 933 : cluster 0 pgmap v1514: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2024-03-07T03:45:05.068 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:04 smithi195 bash[66542]: cluster 2024-03-07T03:45:03.140218+0000 mgr.y (mgr.34101) 933 : cluster 0 pgmap v1514: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.0 MiB/s wr, 17 op/s 2024-03-07T03:45:07.046 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:06 smithi148 bash[82222]: cluster 2024-03-07T03:45:05.141095+0000 mgr.y (mgr.34101) 934 : cluster 0 pgmap v1515: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-03-07T03:45:07.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:06 smithi148 bash[82222]: cluster 2024-03-07T03:45:05.757392+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2024-03-07T03:45:07.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:06 smithi148 bash[79819]: cluster 2024-03-07T03:45:05.141095+0000 mgr.y (mgr.34101) 934 : cluster 0 pgmap v1515: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-03-07T03:45:07.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:06 smithi148 bash[79819]: cluster 2024-03-07T03:45:05.757392+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2024-03-07T03:45:07.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:06 smithi195 bash[66542]: cluster 2024-03-07T03:45:05.141095+0000 mgr.y (mgr.34101) 934 : cluster 0 pgmap v1515: 105 pgs: 105 active+clean; 308 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-03-07T03:45:07.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:06 smithi195 bash[66542]: cluster 2024-03-07T03:45:05.757392+0000 mon.a (mon.0) 1739 : cluster 0 osdmap e963: 8 total, 8 up, 8 in 2024-03-07T03:45:07.546 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:45:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:45:07] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:45:09.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:08 smithi148 bash[79819]: cluster 2024-03-07T03:45:07.141730+0000 mgr.y (mgr.34101) 935 : cluster 0 pgmap v1517: 105 pgs: 105 active+clean; 324 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-03-07T03:45:09.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:08 smithi148 bash[82222]: cluster 2024-03-07T03:45:07.141730+0000 mgr.y (mgr.34101) 935 : cluster 0 pgmap v1517: 105 pgs: 105 active+clean; 324 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-03-07T03:45:09.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:08 smithi195 bash[66542]: cluster 2024-03-07T03:45:07.141730+0000 mgr.y (mgr.34101) 935 : cluster 0 pgmap v1517: 105 pgs: 105 active+clean; 324 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 9.0 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-03-07T03:45:11.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:10 smithi148 bash[79819]: cluster 2024-03-07T03:45:09.142521+0000 mgr.y (mgr.34101) 936 : cluster 0 pgmap v1518: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:11.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:10 smithi148 bash[82222]: cluster 2024-03-07T03:45:09.142521+0000 mgr.y (mgr.34101) 936 : cluster 0 pgmap v1518: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:11.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:10 smithi195 bash[66542]: cluster 2024-03-07T03:45:09.142521+0000 mgr.y (mgr.34101) 936 : cluster 0 pgmap v1518: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:12.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:45:12 smithi195 bash[59134]: ts=2024-03-07T03:45:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:13.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:12 smithi148 bash[79819]: cluster 2024-03-07T03:45:11.143148+0000 mgr.y (mgr.34101) 937 : cluster 0 pgmap v1519: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:13.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:12 smithi148 bash[79819]: audit 2024-03-07T03:45:12.344942+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:13.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:12 smithi148 bash[82222]: cluster 2024-03-07T03:45:11.143148+0000 mgr.y (mgr.34101) 937 : cluster 0 pgmap v1519: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:13.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:12 smithi148 bash[82222]: audit 2024-03-07T03:45:12.344942+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:13.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:12 smithi195 bash[66542]: cluster 2024-03-07T03:45:11.143148+0000 mgr.y (mgr.34101) 937 : cluster 0 pgmap v1519: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:13.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:12 smithi195 bash[66542]: audit 2024-03-07T03:45:12.344942+0000 mon.a (mon.0) 1740 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:15.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:14 smithi148 bash[79819]: cluster 2024-03-07T03:45:13.143737+0000 mgr.y (mgr.34101) 938 : cluster 0 pgmap v1520: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:15.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:14 smithi148 bash[82222]: cluster 2024-03-07T03:45:13.143737+0000 mgr.y (mgr.34101) 938 : cluster 0 pgmap v1520: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:15.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:14 smithi195 bash[66542]: cluster 2024-03-07T03:45:13.143737+0000 mgr.y (mgr.34101) 938 : cluster 0 pgmap v1520: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:17.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:16 smithi148 bash[79819]: cluster 2024-03-07T03:45:15.144526+0000 mgr.y (mgr.34101) 939 : cluster 0 pgmap v1521: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:45:17.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:16 smithi148 bash[82222]: cluster 2024-03-07T03:45:15.144526+0000 mgr.y (mgr.34101) 939 : cluster 0 pgmap v1521: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:45:17.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:16 smithi195 bash[66542]: cluster 2024-03-07T03:45:15.144526+0000 mgr.y (mgr.34101) 939 : cluster 0 pgmap v1521: 105 pgs: 105 active+clean; 349 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 4.1 MiB/s wr, 26 op/s 2024-03-07T03:45:17.546 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:45:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:45:17] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-03-07T03:45:19.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:18 smithi148 bash[79819]: cluster 2024-03-07T03:45:17.145090+0000 mgr.y (mgr.34101) 940 : cluster 0 pgmap v1522: 105 pgs: 105 active+clean; 353 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 23 op/s 2024-03-07T03:45:19.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:18 smithi148 bash[82222]: cluster 2024-03-07T03:45:17.145090+0000 mgr.y (mgr.34101) 940 : cluster 0 pgmap v1522: 105 pgs: 105 active+clean; 353 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 23 op/s 2024-03-07T03:45:19.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:18 smithi195 bash[66542]: cluster 2024-03-07T03:45:17.145090+0000 mgr.y (mgr.34101) 940 : cluster 0 pgmap v1522: 105 pgs: 105 active+clean; 353 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 3.6 MiB/s wr, 23 op/s 2024-03-07T03:45:21.046 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:20 smithi148 bash[79819]: cluster 2024-03-07T03:45:19.145825+0000 mgr.y (mgr.34101) 941 : cluster 0 pgmap v1523: 105 pgs: 105 active+clean; 353 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-03-07T03:45:21.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:20 smithi148 bash[82222]: cluster 2024-03-07T03:45:19.145825+0000 mgr.y (mgr.34101) 941 : cluster 0 pgmap v1523: 105 pgs: 105 active+clean; 353 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-03-07T03:45:21.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:20 smithi195 bash[66542]: cluster 2024-03-07T03:45:19.145825+0000 mgr.y (mgr.34101) 941 : cluster 0 pgmap v1523: 105 pgs: 105 active+clean; 353 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 2.4 MiB/s wr, 30 op/s 2024-03-07T03:45:22.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:21 smithi148 bash[79819]: cluster 2024-03-07T03:45:20.790674+0000 mon.a (mon.0) 1741 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2024-03-07T03:45:22.047 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:21 smithi148 bash[82222]: cluster 2024-03-07T03:45:20.790674+0000 mon.a (mon.0) 1741 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2024-03-07T03:45:22.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:21 smithi195 bash[66542]: cluster 2024-03-07T03:45:20.790674+0000 mon.a (mon.0) 1741 : cluster 0 osdmap e964: 8 total, 8 up, 8 in 2024-03-07T03:45:22.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:45:22 smithi195 bash[59134]: ts=2024-03-07T03:45:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:23.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:22 smithi195 bash[66542]: cluster 2024-03-07T03:45:21.146286+0000 mgr.y (mgr.34101) 942 : cluster 0 pgmap v1525: 105 pgs: 105 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 65 KiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:45:23.136 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/6.ShrinkToOneAfterSnapshot (27893 ms) 2024-03-07T03:45:23.136 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/6.MinOverlap 2024-03-07T03:45:23.136 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:45:23.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:22 smithi148 bash[79819]: cluster 2024-03-07T03:45:21.146286+0000 mgr.y (mgr.34101) 942 : cluster 0 pgmap v1525: 105 pgs: 105 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 65 KiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:45:23.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:22 smithi148 bash[82222]: cluster 2024-03-07T03:45:21.146286+0000 mgr.y (mgr.34101) 942 : cluster 0 pgmap v1525: 105 pgs: 105 active+clean; 365 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 65 KiB/s rd, 1.6 MiB/s wr, 11 op/s 2024-03-07T03:45:25.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:24 smithi195 bash[66542]: cluster 2024-03-07T03:45:23.146985+0000 mgr.y (mgr.34101) 943 : cluster 0 pgmap v1526: 105 pgs: 105 active+clean; 393 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.4 MiB/s wr, 12 op/s 2024-03-07T03:45:25.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:24 smithi148 bash[79819]: cluster 2024-03-07T03:45:23.146985+0000 mgr.y (mgr.34101) 943 : cluster 0 pgmap v1526: 105 pgs: 105 active+clean; 393 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.4 MiB/s wr, 12 op/s 2024-03-07T03:45:25.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:24 smithi148 bash[82222]: cluster 2024-03-07T03:45:23.146985+0000 mgr.y (mgr.34101) 943 : cluster 0 pgmap v1526: 105 pgs: 105 active+clean; 393 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.4 MiB/s wr, 12 op/s 2024-03-07T03:45:27.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:26 smithi195 bash[66542]: cluster 2024-03-07T03:45:25.147811+0000 mgr.y (mgr.34101) 944 : cluster 0 pgmap v1527: 105 pgs: 105 active+clean; 414 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.5 MiB/s wr, 48 op/s 2024-03-07T03:45:27.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:26 smithi148 bash[79819]: cluster 2024-03-07T03:45:25.147811+0000 mgr.y (mgr.34101) 944 : cluster 0 pgmap v1527: 105 pgs: 105 active+clean; 414 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.5 MiB/s wr, 48 op/s 2024-03-07T03:45:27.173 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:26 smithi148 bash[82222]: cluster 2024-03-07T03:45:25.147811+0000 mgr.y (mgr.34101) 944 : cluster 0 pgmap v1527: 105 pgs: 105 active+clean; 414 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.5 MiB/s wr, 48 op/s 2024-03-07T03:45:27.547 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:45:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:45:27] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-03-07T03:45:28.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:27 smithi195 bash[66542]: audit 2024-03-07T03:45:27.345261+0000 mon.a (mon.0) 1742 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:28.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:27 smithi148 bash[79819]: audit 2024-03-07T03:45:27.345261+0000 mon.a (mon.0) 1742 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:28.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:27 smithi148 bash[82222]: audit 2024-03-07T03:45:27.345261+0000 mon.a (mon.0) 1742 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:29.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:28 smithi195 bash[66542]: cluster 2024-03-07T03:45:27.148819+0000 mgr.y (mgr.34101) 945 : cluster 0 pgmap v1528: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.1 MiB/s wr, 49 op/s 2024-03-07T03:45:29.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:28 smithi148 bash[79819]: cluster 2024-03-07T03:45:27.148819+0000 mgr.y (mgr.34101) 945 : cluster 0 pgmap v1528: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.1 MiB/s wr, 49 op/s 2024-03-07T03:45:29.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:28 smithi148 bash[82222]: cluster 2024-03-07T03:45:27.148819+0000 mgr.y (mgr.34101) 945 : cluster 0 pgmap v1528: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.1 MiB/s wr, 49 op/s 2024-03-07T03:45:31.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:30 smithi148 bash[79819]: cluster 2024-03-07T03:45:29.149590+0000 mgr.y (mgr.34101) 946 : cluster 0 pgmap v1529: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.1 MiB/s wr, 39 op/s 2024-03-07T03:45:31.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:30 smithi148 bash[82222]: cluster 2024-03-07T03:45:29.149590+0000 mgr.y (mgr.34101) 946 : cluster 0 pgmap v1529: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.1 MiB/s wr, 39 op/s 2024-03-07T03:45:31.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:30 smithi195 bash[66542]: cluster 2024-03-07T03:45:29.149590+0000 mgr.y (mgr.34101) 946 : cluster 0 pgmap v1529: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 8.1 MiB/s wr, 39 op/s 2024-03-07T03:45:32.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:45:32 smithi195 bash[59134]: ts=2024-03-07T03:45:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:33.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:32 smithi148 bash[79819]: cluster 2024-03-07T03:45:31.150451+0000 mgr.y (mgr.34101) 947 : cluster 0 pgmap v1530: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 6.7 MiB/s wr, 37 op/s 2024-03-07T03:45:33.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:32 smithi148 bash[79819]: cluster 2024-03-07T03:45:31.835020+0000 mon.a (mon.0) 1743 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2024-03-07T03:45:33.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:32 smithi148 bash[82222]: cluster 2024-03-07T03:45:31.150451+0000 mgr.y (mgr.34101) 947 : cluster 0 pgmap v1530: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 6.7 MiB/s wr, 37 op/s 2024-03-07T03:45:33.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:32 smithi148 bash[82222]: cluster 2024-03-07T03:45:31.835020+0000 mon.a (mon.0) 1743 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2024-03-07T03:45:33.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:32 smithi195 bash[66542]: cluster 2024-03-07T03:45:31.150451+0000 mgr.y (mgr.34101) 947 : cluster 0 pgmap v1530: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 6.7 MiB/s wr, 37 op/s 2024-03-07T03:45:33.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:32 smithi195 bash[66542]: cluster 2024-03-07T03:45:31.835020+0000 mon.a (mon.0) 1743 : cluster 0 osdmap e965: 8 total, 8 up, 8 in 2024-03-07T03:45:35.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:34 smithi148 bash[79819]: cluster 2024-03-07T03:45:33.151003+0000 mgr.y (mgr.34101) 948 : cluster 0 pgmap v1532: 105 pgs: 105 active+clean; 438 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:45:35.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:34 smithi148 bash[82222]: cluster 2024-03-07T03:45:33.151003+0000 mgr.y (mgr.34101) 948 : cluster 0 pgmap v1532: 105 pgs: 105 active+clean; 438 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:45:35.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:34 smithi195 bash[66542]: cluster 2024-03-07T03:45:33.151003+0000 mgr.y (mgr.34101) 948 : cluster 0 pgmap v1532: 105 pgs: 105 active+clean; 438 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:45:37.172 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:36 smithi148 bash[79819]: cluster 2024-03-07T03:45:35.151820+0000 mgr.y (mgr.34101) 949 : cluster 0 pgmap v1533: 105 pgs: 105 active+clean; 471 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 846 KiB/s rd, 5.7 MiB/s wr, 36 op/s 2024-03-07T03:45:37.172 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:36 smithi148 bash[82222]: cluster 2024-03-07T03:45:35.151820+0000 mgr.y (mgr.34101) 949 : cluster 0 pgmap v1533: 105 pgs: 105 active+clean; 471 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 846 KiB/s rd, 5.7 MiB/s wr, 36 op/s 2024-03-07T03:45:37.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:36 smithi195 bash[66542]: cluster 2024-03-07T03:45:35.151820+0000 mgr.y (mgr.34101) 949 : cluster 0 pgmap v1533: 105 pgs: 105 active+clean; 471 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 846 KiB/s rd, 5.7 MiB/s wr, 36 op/s 2024-03-07T03:45:37.547 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:45:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:45:37] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:45:39.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:38 smithi148 bash[79819]: cluster 2024-03-07T03:45:37.152461+0000 mgr.y (mgr.34101) 950 : cluster 0 pgmap v1534: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:39.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:38 smithi148 bash[82222]: cluster 2024-03-07T03:45:37.152461+0000 mgr.y (mgr.34101) 950 : cluster 0 pgmap v1534: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:39.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:38 smithi195 bash[66542]: cluster 2024-03-07T03:45:37.152461+0000 mgr.y (mgr.34101) 950 : cluster 0 pgmap v1534: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:41.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:40 smithi148 bash[79819]: cluster 2024-03-07T03:45:39.153214+0000 mgr.y (mgr.34101) 951 : cluster 0 pgmap v1535: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:41.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:40 smithi148 bash[82222]: cluster 2024-03-07T03:45:39.153214+0000 mgr.y (mgr.34101) 951 : cluster 0 pgmap v1535: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:41.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:40 smithi195 bash[66542]: cluster 2024-03-07T03:45:39.153214+0000 mgr.y (mgr.34101) 951 : cluster 0 pgmap v1535: 105 pgs: 105 active+clean; 475 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:45:42.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:41 smithi148 bash[82222]: cluster 2024-03-07T03:45:41.153800+0000 mgr.y (mgr.34101) 952 : cluster 0 pgmap v1536: 105 pgs: 105 active+clean; 479 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:45:42.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:41 smithi148 bash[79819]: cluster 2024-03-07T03:45:41.153800+0000 mgr.y (mgr.34101) 952 : cluster 0 pgmap v1536: 105 pgs: 105 active+clean; 479 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:45:42.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:41 smithi195 bash[66542]: cluster 2024-03-07T03:45:41.153800+0000 mgr.y (mgr.34101) 952 : cluster 0 pgmap v1536: 105 pgs: 105 active+clean; 479 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:45:42.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:45:42 smithi195 bash[59134]: ts=2024-03-07T03:45:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:43.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:42 smithi148 bash[79819]: audit 2024-03-07T03:45:42.345565+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:43.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:42 smithi148 bash[82222]: audit 2024-03-07T03:45:42.345565+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:43.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:42 smithi195 bash[66542]: audit 2024-03-07T03:45:42.345565+0000 mon.a (mon.0) 1744 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:44.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:43 smithi148 bash[79819]: cluster 2024-03-07T03:45:43.154608+0000 mgr.y (mgr.34101) 953 : cluster 0 pgmap v1537: 105 pgs: 105 active+clean; 483 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 35 op/s 2024-03-07T03:45:44.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:43 smithi148 bash[82222]: cluster 2024-03-07T03:45:43.154608+0000 mgr.y (mgr.34101) 953 : cluster 0 pgmap v1537: 105 pgs: 105 active+clean; 483 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 35 op/s 2024-03-07T03:45:44.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:43 smithi195 bash[66542]: cluster 2024-03-07T03:45:43.154608+0000 mgr.y (mgr.34101) 953 : cluster 0 pgmap v1537: 105 pgs: 105 active+clean; 483 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 4.0 MiB/s wr, 35 op/s 2024-03-07T03:45:45.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:44 smithi148 bash[79819]: audit 2024-03-07T03:45:44.782859+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:45:45.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:44 smithi148 bash[82222]: audit 2024-03-07T03:45:44.782859+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:45:45.309 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/6.MinOverlap (22173 ms) 2024-03-07T03:45:45.309 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/6.ZeroOverlap 2024-03-07T03:45:45.309 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:45:45.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:44 smithi195 bash[66542]: audit 2024-03-07T03:45:44.782859+0000 mon.a (mon.0) 1745 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:45:46.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:45 smithi148 bash[79819]: cluster 2024-03-07T03:45:45.155508+0000 mgr.y (mgr.34101) 954 : cluster 0 pgmap v1538: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.8 MiB/s wr, 49 op/s 2024-03-07T03:45:46.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:45 smithi148 bash[82222]: cluster 2024-03-07T03:45:45.155508+0000 mgr.y (mgr.34101) 954 : cluster 0 pgmap v1538: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.8 MiB/s wr, 49 op/s 2024-03-07T03:45:46.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:45 smithi195 bash[66542]: cluster 2024-03-07T03:45:45.155508+0000 mgr.y (mgr.34101) 954 : cluster 0 pgmap v1538: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 5.8 MiB/s wr, 49 op/s 2024-03-07T03:45:47.547 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:45:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:45:47] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-03-07T03:45:48.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:48 smithi148 bash[79819]: cluster 2024-03-07T03:45:47.156217+0000 mgr.y (mgr.34101) 955 : cluster 0 pgmap v1539: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.7 MiB/s wr, 24 op/s 2024-03-07T03:45:48.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:48 smithi148 bash[82222]: cluster 2024-03-07T03:45:47.156217+0000 mgr.y (mgr.34101) 955 : cluster 0 pgmap v1539: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.7 MiB/s wr, 24 op/s 2024-03-07T03:45:48.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:48 smithi195 bash[66542]: cluster 2024-03-07T03:45:47.156217+0000 mgr.y (mgr.34101) 955 : cluster 0 pgmap v1539: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.7 MiB/s wr, 24 op/s 2024-03-07T03:45:50.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:50 smithi148 bash[79819]: cluster 2024-03-07T03:45:49.156987+0000 mgr.y (mgr.34101) 956 : cluster 0 pgmap v1540: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:45:50.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:50 smithi148 bash[82222]: cluster 2024-03-07T03:45:49.156987+0000 mgr.y (mgr.34101) 956 : cluster 0 pgmap v1540: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:45:50.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:50 smithi195 bash[66542]: cluster 2024-03-07T03:45:49.156987+0000 mgr.y (mgr.34101) 956 : cluster 0 pgmap v1540: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:45:52.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[79819]: audit 2024-03-07T03:45:50.721266+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[79819]: audit 2024-03-07T03:45:50.727202+0000 mon.a (mon.0) 1747 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[79819]: audit 2024-03-07T03:45:50.818388+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[79819]: audit 2024-03-07T03:45:50.824824+0000 mon.a (mon.0) 1749 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[79819]: audit 2024-03-07T03:45:51.469228+0000 mon.a (mon.0) 1750 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[79819]: audit 2024-03-07T03:45:51.470305+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[79819]: audit 2024-03-07T03:45:51.477385+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[82222]: audit 2024-03-07T03:45:50.721266+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[82222]: audit 2024-03-07T03:45:50.727202+0000 mon.a (mon.0) 1747 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[82222]: audit 2024-03-07T03:45:50.818388+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[82222]: audit 2024-03-07T03:45:50.824824+0000 mon.a (mon.0) 1749 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[82222]: audit 2024-03-07T03:45:51.469228+0000 mon.a (mon.0) 1750 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[82222]: audit 2024-03-07T03:45:51.470305+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:45:52.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:51 smithi148 bash[82222]: audit 2024-03-07T03:45:51.477385+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:51 smithi195 bash[66542]: audit 2024-03-07T03:45:50.721266+0000 mon.a (mon.0) 1746 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:51 smithi195 bash[66542]: audit 2024-03-07T03:45:50.727202+0000 mon.a (mon.0) 1747 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:51 smithi195 bash[66542]: audit 2024-03-07T03:45:50.818388+0000 mon.a (mon.0) 1748 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:51 smithi195 bash[66542]: audit 2024-03-07T03:45:50.824824+0000 mon.a (mon.0) 1749 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:51 smithi195 bash[66542]: audit 2024-03-07T03:45:51.469228+0000 mon.a (mon.0) 1750 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:45:52.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:51 smithi195 bash[66542]: audit 2024-03-07T03:45:51.470305+0000 mon.a (mon.0) 1751 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:45:52.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:51 smithi195 bash[66542]: audit 2024-03-07T03:45:51.477385+0000 mon.a (mon.0) 1752 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:45:52.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:45:52 smithi195 bash[59134]: ts=2024-03-07T03:45:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:45:53.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:52 smithi148 bash[79819]: cluster 2024-03-07T03:45:51.157493+0000 mgr.y (mgr.34101) 957 : cluster 0 pgmap v1541: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:45:53.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:52 smithi148 bash[82222]: cluster 2024-03-07T03:45:51.157493+0000 mgr.y (mgr.34101) 957 : cluster 0 pgmap v1541: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:45:53.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:52 smithi195 bash[66542]: cluster 2024-03-07T03:45:51.157493+0000 mgr.y (mgr.34101) 957 : cluster 0 pgmap v1541: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-03-07T03:45:55.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:54 smithi148 bash[79819]: cluster 2024-03-07T03:45:53.158218+0000 mgr.y (mgr.34101) 958 : cluster 0 pgmap v1542: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-03-07T03:45:55.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:54 smithi148 bash[79819]: cluster 2024-03-07T03:45:53.841503+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2024-03-07T03:45:55.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:54 smithi148 bash[82222]: cluster 2024-03-07T03:45:53.158218+0000 mgr.y (mgr.34101) 958 : cluster 0 pgmap v1542: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-03-07T03:45:55.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:54 smithi148 bash[82222]: cluster 2024-03-07T03:45:53.841503+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2024-03-07T03:45:55.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:54 smithi195 bash[66542]: cluster 2024-03-07T03:45:53.158218+0000 mgr.y (mgr.34101) 958 : cluster 0 pgmap v1542: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-03-07T03:45:55.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:54 smithi195 bash[66542]: cluster 2024-03-07T03:45:53.841503+0000 mon.a (mon.0) 1753 : cluster 0 osdmap e966: 8 total, 8 up, 8 in 2024-03-07T03:45:57.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:56 smithi148 bash[79819]: cluster 2024-03-07T03:45:55.158919+0000 mgr.y (mgr.34101) 959 : cluster 0 pgmap v1544: 105 pgs: 105 active+clean; 520 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 17 op/s 2024-03-07T03:45:57.173 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:56 smithi148 bash[82222]: cluster 2024-03-07T03:45:55.158919+0000 mgr.y (mgr.34101) 959 : cluster 0 pgmap v1544: 105 pgs: 105 active+clean; 520 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 17 op/s 2024-03-07T03:45:57.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:56 smithi195 bash[66542]: cluster 2024-03-07T03:45:55.158919+0000 mgr.y (mgr.34101) 959 : cluster 0 pgmap v1544: 105 pgs: 105 active+clean; 520 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.6 MiB/s rd, 3.3 MiB/s wr, 17 op/s 2024-03-07T03:45:57.547 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:45:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:45:57] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-03-07T03:45:58.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:57 smithi148 bash[79819]: audit 2024-03-07T03:45:57.346011+0000 mon.a (mon.0) 1754 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:58.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:57 smithi148 bash[82222]: audit 2024-03-07T03:45:57.346011+0000 mon.a (mon.0) 1754 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:58.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:57 smithi195 bash[66542]: audit 2024-03-07T03:45:57.346011+0000 mon.a (mon.0) 1754 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:45:59.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:58 smithi148 bash[79819]: cluster 2024-03-07T03:45:57.159588+0000 mgr.y (mgr.34101) 960 : cluster 0 pgmap v1545: 105 pgs: 105 active+clean; 536 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:45:59.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:58 smithi148 bash[82222]: cluster 2024-03-07T03:45:57.159588+0000 mgr.y (mgr.34101) 960 : cluster 0 pgmap v1545: 105 pgs: 105 active+clean; 536 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:45:59.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:58 smithi195 bash[66542]: cluster 2024-03-07T03:45:57.159588+0000 mgr.y (mgr.34101) 960 : cluster 0 pgmap v1545: 105 pgs: 105 active+clean; 536 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:46:00.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:45:59 smithi148 bash[79819]: cluster 2024-03-07T03:45:59.160396+0000 mgr.y (mgr.34101) 961 : cluster 0 pgmap v1546: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:00.297 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:45:59 smithi148 bash[82222]: cluster 2024-03-07T03:45:59.160396+0000 mgr.y (mgr.34101) 961 : cluster 0 pgmap v1546: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:00.319 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:45:59 smithi195 bash[66542]: cluster 2024-03-07T03:45:59.160396+0000 mgr.y (mgr.34101) 961 : cluster 0 pgmap v1546: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 436 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:02.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:02 smithi148 bash[79819]: cluster 2024-03-07T03:46:01.161005+0000 mgr.y (mgr.34101) 962 : cluster 0 pgmap v1547: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:02.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:02 smithi148 bash[82222]: cluster 2024-03-07T03:46:01.161005+0000 mgr.y (mgr.34101) 962 : cluster 0 pgmap v1547: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:02.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:02 smithi195 bash[66542]: cluster 2024-03-07T03:46:01.161005+0000 mgr.y (mgr.34101) 962 : cluster 0 pgmap v1547: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:02.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:46:02 smithi195 bash[59134]: ts=2024-03-07T03:46:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:04.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:04 smithi148 bash[79819]: cluster 2024-03-07T03:46:03.161576+0000 mgr.y (mgr.34101) 963 : cluster 0 pgmap v1548: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:04.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:04 smithi148 bash[82222]: cluster 2024-03-07T03:46:03.161576+0000 mgr.y (mgr.34101) 963 : cluster 0 pgmap v1548: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:04.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:04 smithi195 bash[66542]: cluster 2024-03-07T03:46:03.161576+0000 mgr.y (mgr.34101) 963 : cluster 0 pgmap v1548: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:06.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:06 smithi148 bash[79819]: cluster 2024-03-07T03:46:05.162734+0000 mgr.y (mgr.34101) 964 : cluster 0 pgmap v1549: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 4.0 MiB/s wr, 31 op/s 2024-03-07T03:46:06.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:06 smithi148 bash[82222]: cluster 2024-03-07T03:46:05.162734+0000 mgr.y (mgr.34101) 964 : cluster 0 pgmap v1549: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 4.0 MiB/s wr, 31 op/s 2024-03-07T03:46:06.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:06 smithi195 bash[66542]: cluster 2024-03-07T03:46:05.162734+0000 mgr.y (mgr.34101) 964 : cluster 0 pgmap v1549: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 4.0 MiB/s wr, 31 op/s 2024-03-07T03:46:07.098 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/6.ZeroOverlap (21788 ms) 2024-03-07T03:46:07.247 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/6 (162233 ms total) 2024-03-07T03:46:07.247 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:46:07.247 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/7, where TypeParam = LUKS2UnderLUKS1 2024-03-07T03:46:07.292 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/7.Simple 2024-03-07T03:46:07.547 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:46:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:46:07] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-03-07T03:46:08.547 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:08 smithi148 bash[82222]: cluster 2024-03-07T03:46:07.163734+0000 mgr.y (mgr.34101) 965 : cluster 0 pgmap v1550: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 2.7 MiB/s wr, 38 op/s 2024-03-07T03:46:08.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:08 smithi148 bash[82222]: cluster 2024-03-07T03:46:07.237633+0000 mon.a (mon.0) 1755 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2024-03-07T03:46:08.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:08 smithi148 bash[79819]: cluster 2024-03-07T03:46:07.163734+0000 mgr.y (mgr.34101) 965 : cluster 0 pgmap v1550: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 2.7 MiB/s wr, 38 op/s 2024-03-07T03:46:08.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:08 smithi148 bash[79819]: cluster 2024-03-07T03:46:07.237633+0000 mon.a (mon.0) 1755 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2024-03-07T03:46:08.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:08 smithi195 bash[66542]: cluster 2024-03-07T03:46:07.163734+0000 mgr.y (mgr.34101) 965 : cluster 0 pgmap v1550: 105 pgs: 105 active+clean; 552 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 2.7 MiB/s wr, 38 op/s 2024-03-07T03:46:08.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:08 smithi195 bash[66542]: cluster 2024-03-07T03:46:07.237633+0000 mon.a (mon.0) 1755 : cluster 0 osdmap e967: 8 total, 8 up, 8 in 2024-03-07T03:46:09.264 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:46:09.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:09 smithi148 bash[79819]: cluster 2024-03-07T03:46:08.252507+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2024-03-07T03:46:09.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:09 smithi148 bash[79819]: audit 2024-03-07T03:46:08.256079+0000 mon.a (mon.0) 1757 : audit 1 from='client.? 172.21.15.148:0/2224833281' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-43","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:46:09.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:09 smithi148 bash[82222]: cluster 2024-03-07T03:46:08.252507+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2024-03-07T03:46:09.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:09 smithi148 bash[82222]: audit 2024-03-07T03:46:08.256079+0000 mon.a (mon.0) 1757 : audit 1 from='client.? 172.21.15.148:0/2224833281' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-43","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:46:09.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:09 smithi195 bash[66542]: cluster 2024-03-07T03:46:08.252507+0000 mon.a (mon.0) 1756 : cluster 0 osdmap e968: 8 total, 8 up, 8 in 2024-03-07T03:46:09.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:09 smithi195 bash[66542]: audit 2024-03-07T03:46:08.256079+0000 mon.a (mon.0) 1757 : audit 1 from='client.? 172.21.15.148:0/2224833281' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-43","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:46:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[79819]: cluster 2024-03-07T03:46:09.164911+0000 mgr.y (mgr.34101) 966 : cluster 0 pgmap v1553: 105 pgs: 20 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:46:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[79819]: cluster 2024-03-07T03:46:09.244921+0000 mon.a (mon.0) 1758 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:46:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[79819]: audit 2024-03-07T03:46:09.254711+0000 mon.a (mon.0) 1759 : audit 1 from='client.? 172.21.15.148:0/2224833281' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:46:10.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[79819]: cluster 2024-03-07T03:46:09.264892+0000 mon.a (mon.0) 1760 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2024-03-07T03:46:10.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[82222]: cluster 2024-03-07T03:46:09.164911+0000 mgr.y (mgr.34101) 966 : cluster 0 pgmap v1553: 105 pgs: 20 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:46:10.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[82222]: cluster 2024-03-07T03:46:09.244921+0000 mon.a (mon.0) 1758 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:46:10.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[82222]: audit 2024-03-07T03:46:09.254711+0000 mon.a (mon.0) 1759 : audit 1 from='client.? 172.21.15.148:0/2224833281' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:46:10.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:10 smithi148 bash[82222]: cluster 2024-03-07T03:46:09.264892+0000 mon.a (mon.0) 1760 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2024-03-07T03:46:10.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:10 smithi195 bash[66542]: cluster 2024-03-07T03:46:09.164911+0000 mgr.y (mgr.34101) 966 : cluster 0 pgmap v1553: 105 pgs: 20 creating+peering, 12 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:46:10.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:10 smithi195 bash[66542]: cluster 2024-03-07T03:46:09.244921+0000 mon.a (mon.0) 1758 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:46:10.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:10 smithi195 bash[66542]: audit 2024-03-07T03:46:09.254711+0000 mon.a (mon.0) 1759 : audit 1 from='client.? 172.21.15.148:0/2224833281' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:46:10.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:10 smithi195 bash[66542]: cluster 2024-03-07T03:46:09.264892+0000 mon.a (mon.0) 1760 : cluster 0 osdmap e969: 8 total, 8 up, 8 in 2024-03-07T03:46:11.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:11 smithi148 bash[79819]: cluster 2024-03-07T03:46:10.263218+0000 mon.a (mon.0) 1761 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2024-03-07T03:46:11.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:11 smithi148 bash[82222]: cluster 2024-03-07T03:46:10.263218+0000 mon.a (mon.0) 1761 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2024-03-07T03:46:11.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:11 smithi195 bash[66542]: cluster 2024-03-07T03:46:10.263218+0000 mon.a (mon.0) 1761 : cluster 0 osdmap e970: 8 total, 8 up, 8 in 2024-03-07T03:46:12.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:12 smithi148 bash[79819]: cluster 2024-03-07T03:46:11.165592+0000 mgr.y (mgr.34101) 967 : cluster 0 pgmap v1556: 105 pgs: 20 creating+peering, 7 unknown, 78 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:46:12.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:12 smithi148 bash[79819]: cluster 2024-03-07T03:46:11.269793+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:46:12.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:12 smithi148 bash[79819]: cluster 2024-03-07T03:46:11.278156+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2024-03-07T03:46:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:12 smithi148 bash[82222]: cluster 2024-03-07T03:46:11.165592+0000 mgr.y (mgr.34101) 967 : cluster 0 pgmap v1556: 105 pgs: 20 creating+peering, 7 unknown, 78 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:46:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:12 smithi148 bash[82222]: cluster 2024-03-07T03:46:11.269793+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:46:12.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:12 smithi148 bash[82222]: cluster 2024-03-07T03:46:11.278156+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2024-03-07T03:46:12.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:46:12 smithi195 bash[59134]: ts=2024-03-07T03:46:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:12.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:12 smithi195 bash[66542]: cluster 2024-03-07T03:46:11.165592+0000 mgr.y (mgr.34101) 967 : cluster 0 pgmap v1556: 105 pgs: 20 creating+peering, 7 unknown, 78 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:46:12.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:12 smithi195 bash[66542]: cluster 2024-03-07T03:46:11.269793+0000 mon.a (mon.0) 1762 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:46:12.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:12 smithi195 bash[66542]: cluster 2024-03-07T03:46:11.278156+0000 mon.a (mon.0) 1763 : cluster 0 osdmap e971: 8 total, 8 up, 8 in 2024-03-07T03:46:13.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:13 smithi148 bash[79819]: audit 2024-03-07T03:46:12.346452+0000 mon.a (mon.0) 1764 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:13.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:13 smithi148 bash[82222]: audit 2024-03-07T03:46:12.346452+0000 mon.a (mon.0) 1764 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:13.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:13 smithi195 bash[66542]: audit 2024-03-07T03:46:12.346452+0000 mon.a (mon.0) 1764 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:14.548 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:14 smithi148 bash[79819]: cluster 2024-03-07T03:46:13.166203+0000 mgr.y (mgr.34101) 968 : cluster 0 pgmap v1558: 105 pgs: 20 creating+peering, 85 active+clean; 582 KiB data, 815 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.0 KiB/s wr, 8 op/s 2024-03-07T03:46:14.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:14 smithi148 bash[82222]: cluster 2024-03-07T03:46:13.166203+0000 mgr.y (mgr.34101) 968 : cluster 0 pgmap v1558: 105 pgs: 20 creating+peering, 85 active+clean; 582 KiB data, 815 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.0 KiB/s wr, 8 op/s 2024-03-07T03:46:14.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:14 smithi195 bash[66542]: cluster 2024-03-07T03:46:13.166203+0000 mgr.y (mgr.34101) 968 : cluster 0 pgmap v1558: 105 pgs: 20 creating+peering, 85 active+clean; 582 KiB data, 815 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 1.0 KiB/s wr, 8 op/s 2024-03-07T03:46:16.547 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:16 smithi148 bash[79819]: cluster 2024-03-07T03:46:15.166998+0000 mgr.y (mgr.34101) 969 : cluster 0 pgmap v1559: 105 pgs: 105 active+clean; 582 KiB data, 617 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-03-07T03:46:16.548 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:16 smithi148 bash[82222]: cluster 2024-03-07T03:46:15.166998+0000 mgr.y (mgr.34101) 969 : cluster 0 pgmap v1559: 105 pgs: 105 active+clean; 582 KiB data, 617 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-03-07T03:46:16.569 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:16 smithi195 bash[66542]: cluster 2024-03-07T03:46:15.166998+0000 mgr.y (mgr.34101) 969 : cluster 0 pgmap v1559: 105 pgs: 105 active+clean; 582 KiB data, 617 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-03-07T03:46:17.547 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:46:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:46:17] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.43.0" 2024-03-07T03:46:19.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:18 smithi148 bash[79819]: cluster 2024-03-07T03:46:17.167634+0000 mgr.y (mgr.34101) 970 : cluster 0 pgmap v1560: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:46:19.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:18 smithi148 bash[79819]: cluster 2024-03-07T03:46:17.733568+0000 mon.a (mon.0) 1765 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2024-03-07T03:46:19.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:18 smithi148 bash[82222]: cluster 2024-03-07T03:46:17.167634+0000 mgr.y (mgr.34101) 970 : cluster 0 pgmap v1560: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:46:19.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:18 smithi148 bash[82222]: cluster 2024-03-07T03:46:17.733568+0000 mon.a (mon.0) 1765 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2024-03-07T03:46:19.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:18 smithi195 bash[66542]: cluster 2024-03-07T03:46:17.167634+0000 mgr.y (mgr.34101) 970 : cluster 0 pgmap v1560: 105 pgs: 105 active+clean; 582 KiB data, 618 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-03-07T03:46:19.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:18 smithi195 bash[66542]: cluster 2024-03-07T03:46:17.733568+0000 mon.a (mon.0) 1765 : cluster 0 osdmap e972: 8 total, 8 up, 8 in 2024-03-07T03:46:20.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:19 smithi148 bash[79819]: cluster 2024-03-07T03:46:18.744120+0000 mon.a (mon.0) 1766 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2024-03-07T03:46:20.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:19 smithi148 bash[79819]: audit 2024-03-07T03:46:18.774668+0000 mon.a (mon.0) 1767 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:20.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:19 smithi148 bash[79819]: audit 2024-03-07T03:46:18.893338+0000 mon.a (mon.0) 1768 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.10", "id": [1, 2]}]: dispatch 2024-03-07T03:46:20.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:19 smithi148 bash[82222]: cluster 2024-03-07T03:46:18.744120+0000 mon.a (mon.0) 1766 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2024-03-07T03:46:20.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:19 smithi148 bash[82222]: audit 2024-03-07T03:46:18.774668+0000 mon.a (mon.0) 1767 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:20.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:19 smithi148 bash[82222]: audit 2024-03-07T03:46:18.893338+0000 mon.a (mon.0) 1768 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.10", "id": [1, 2]}]: dispatch 2024-03-07T03:46:20.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:19 smithi195 bash[66542]: cluster 2024-03-07T03:46:18.744120+0000 mon.a (mon.0) 1766 : cluster 0 osdmap e973: 8 total, 8 up, 8 in 2024-03-07T03:46:20.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:19 smithi195 bash[66542]: audit 2024-03-07T03:46:18.774668+0000 mon.a (mon.0) 1767 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:20.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:19 smithi195 bash[66542]: audit 2024-03-07T03:46:18.893338+0000 mon.a (mon.0) 1768 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.10", "id": [1, 2]}]: dispatch 2024-03-07T03:46:21.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:20 smithi148 bash[79819]: cluster 2024-03-07T03:46:19.168402+0000 mgr.y (mgr.34101) 971 : cluster 0 pgmap v1563: 105 pgs: 105 active+clean; 30 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.7 MiB/s wr, 8 op/s 2024-03-07T03:46:21.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:20 smithi148 bash[79819]: audit 2024-03-07T03:46:19.779631+0000 mon.a (mon.0) 1769 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.10", "id": [1, 2]}]': finished 2024-03-07T03:46:21.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:20 smithi148 bash[79819]: cluster 2024-03-07T03:46:19.786436+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2024-03-07T03:46:21.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:20 smithi148 bash[82222]: cluster 2024-03-07T03:46:19.168402+0000 mgr.y (mgr.34101) 971 : cluster 0 pgmap v1563: 105 pgs: 105 active+clean; 30 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.7 MiB/s wr, 8 op/s 2024-03-07T03:46:21.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:20 smithi148 bash[82222]: audit 2024-03-07T03:46:19.779631+0000 mon.a (mon.0) 1769 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.10", "id": [1, 2]}]': finished 2024-03-07T03:46:21.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:20 smithi148 bash[82222]: cluster 2024-03-07T03:46:19.786436+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2024-03-07T03:46:21.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:20 smithi195 bash[66542]: cluster 2024-03-07T03:46:19.168402+0000 mgr.y (mgr.34101) 971 : cluster 0 pgmap v1563: 105 pgs: 105 active+clean; 30 MiB data, 651 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 3.7 MiB/s wr, 8 op/s 2024-03-07T03:46:21.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:20 smithi195 bash[66542]: audit 2024-03-07T03:46:19.779631+0000 mon.a (mon.0) 1769 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.10", "id": [1, 2]}]': finished 2024-03-07T03:46:21.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:20 smithi195 bash[66542]: cluster 2024-03-07T03:46:19.786436+0000 mon.a (mon.0) 1770 : cluster 0 osdmap e974: 8 total, 8 up, 8 in 2024-03-07T03:46:22.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:21 smithi148 bash[79819]: cluster 2024-03-07T03:46:20.795177+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2024-03-07T03:46:22.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:21 smithi148 bash[82222]: cluster 2024-03-07T03:46:20.795177+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2024-03-07T03:46:22.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:21 smithi195 bash[66542]: cluster 2024-03-07T03:46:20.795177+0000 mon.a (mon.0) 1771 : cluster 0 osdmap e975: 8 total, 8 up, 8 in 2024-03-07T03:46:22.569 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:46:22 smithi195 bash[59134]: ts=2024-03-07T03:46:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:23.047 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:22 smithi148 bash[79819]: cluster 2024-03-07T03:46:21.169015+0000 mgr.y (mgr.34101) 972 : cluster 0 pgmap v1566: 105 pgs: 1 peering, 104 active+clean; 38 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 16 op/s 2024-03-07T03:46:23.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:22 smithi148 bash[82222]: cluster 2024-03-07T03:46:21.169015+0000 mgr.y (mgr.34101) 972 : cluster 0 pgmap v1566: 105 pgs: 1 peering, 104 active+clean; 38 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 16 op/s 2024-03-07T03:46:23.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:22 smithi195 bash[66542]: cluster 2024-03-07T03:46:21.169015+0000 mgr.y (mgr.34101) 972 : cluster 0 pgmap v1566: 105 pgs: 1 peering, 104 active+clean; 38 MiB data, 688 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 9.2 MiB/s wr, 16 op/s 2024-03-07T03:46:25.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:24 smithi148 bash[79819]: cluster 2024-03-07T03:46:23.169781+0000 mgr.y (mgr.34101) 973 : cluster 0 pgmap v1567: 105 pgs: 1 peering, 104 active+clean; 42 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 7.5 MiB/s wr, 62 op/s 2024-03-07T03:46:25.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:24 smithi148 bash[82222]: cluster 2024-03-07T03:46:23.169781+0000 mgr.y (mgr.34101) 973 : cluster 0 pgmap v1567: 105 pgs: 1 peering, 104 active+clean; 42 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 7.5 MiB/s wr, 62 op/s 2024-03-07T03:46:25.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:24 smithi195 bash[66542]: cluster 2024-03-07T03:46:23.169781+0000 mgr.y (mgr.34101) 973 : cluster 0 pgmap v1567: 105 pgs: 1 peering, 104 active+clean; 42 MiB data, 712 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 7.5 MiB/s wr, 62 op/s 2024-03-07T03:46:27.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:26 smithi195 bash[66542]: cluster 2024-03-07T03:46:25.170637+0000 mgr.y (mgr.34101) 974 : cluster 0 pgmap v1568: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 3.7 MiB/s wr, 56 op/s 2024-03-07T03:46:27.174 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:26 smithi148 bash[82222]: cluster 2024-03-07T03:46:25.170637+0000 mgr.y (mgr.34101) 974 : cluster 0 pgmap v1568: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 3.7 MiB/s wr, 56 op/s 2024-03-07T03:46:27.174 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:26 smithi148 bash[79819]: cluster 2024-03-07T03:46:25.170637+0000 mgr.y (mgr.34101) 974 : cluster 0 pgmap v1568: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 86 KiB/s rd, 3.7 MiB/s wr, 56 op/s 2024-03-07T03:46:27.547 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:46:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:46:27] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.43.0" 2024-03-07T03:46:28.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:27 smithi195 bash[66542]: audit 2024-03-07T03:46:27.346559+0000 mon.a (mon.0) 1772 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:28.297 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:27 smithi148 bash[79819]: audit 2024-03-07T03:46:27.346559+0000 mon.a (mon.0) 1772 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:28.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:27 smithi148 bash[82222]: audit 2024-03-07T03:46:27.346559+0000 mon.a (mon.0) 1772 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:29.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:28 smithi195 bash[66542]: cluster 2024-03-07T03:46:27.171470+0000 mgr.y (mgr.34101) 975 : cluster 0 pgmap v1569: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 2.0 MiB/s wr, 45 op/s 2024-03-07T03:46:29.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:28 smithi148 bash[79819]: cluster 2024-03-07T03:46:27.171470+0000 mgr.y (mgr.34101) 975 : cluster 0 pgmap v1569: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 2.0 MiB/s wr, 45 op/s 2024-03-07T03:46:29.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:28 smithi148 bash[82222]: cluster 2024-03-07T03:46:27.171470+0000 mgr.y (mgr.34101) 975 : cluster 0 pgmap v1569: 105 pgs: 105 active+clean; 46 MiB data, 757 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 2.0 MiB/s wr, 45 op/s 2024-03-07T03:46:31.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:30 smithi195 bash[66542]: cluster 2024-03-07T03:46:29.172267+0000 mgr.y (mgr.34101) 976 : cluster 0 pgmap v1570: 105 pgs: 105 active+clean; 50 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 230 KiB/s rd, 2.1 MiB/s wr, 40 op/s 2024-03-07T03:46:31.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:30 smithi148 bash[79819]: cluster 2024-03-07T03:46:29.172267+0000 mgr.y (mgr.34101) 976 : cluster 0 pgmap v1570: 105 pgs: 105 active+clean; 50 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 230 KiB/s rd, 2.1 MiB/s wr, 40 op/s 2024-03-07T03:46:31.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:30 smithi148 bash[82222]: cluster 2024-03-07T03:46:29.172267+0000 mgr.y (mgr.34101) 976 : cluster 0 pgmap v1570: 105 pgs: 105 active+clean; 50 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 230 KiB/s rd, 2.1 MiB/s wr, 40 op/s 2024-03-07T03:46:32.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:46:32 smithi195 bash[59134]: ts=2024-03-07T03:46:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:33.069 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/7.Simple (25776 ms) 2024-03-07T03:46:33.069 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/7.Grow 2024-03-07T03:46:33.070 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:46:33.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:32 smithi148 bash[79819]: cluster 2024-03-07T03:46:31.173010+0000 mgr.y (mgr.34101) 977 : cluster 0 pgmap v1571: 105 pgs: 105 active+clean; 50 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 234 KiB/s rd, 1.2 MiB/s wr, 34 op/s 2024-03-07T03:46:33.373 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:32 smithi148 bash[82222]: cluster 2024-03-07T03:46:31.173010+0000 mgr.y (mgr.34101) 977 : cluster 0 pgmap v1571: 105 pgs: 105 active+clean; 50 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 234 KiB/s rd, 1.2 MiB/s wr, 34 op/s 2024-03-07T03:46:33.373 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:32 smithi195 bash[66542]: cluster 2024-03-07T03:46:31.173010+0000 mgr.y (mgr.34101) 977 : cluster 0 pgmap v1571: 105 pgs: 105 active+clean; 50 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 234 KiB/s rd, 1.2 MiB/s wr, 34 op/s 2024-03-07T03:46:35.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:34 smithi148 bash[79819]: cluster 2024-03-07T03:46:33.173775+0000 mgr.y (mgr.34101) 978 : cluster 0 pgmap v1572: 105 pgs: 105 active+clean; 58 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:46:35.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:34 smithi148 bash[79819]: audit 2024-03-07T03:46:33.795104+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:35.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:34 smithi148 bash[82222]: cluster 2024-03-07T03:46:33.173775+0000 mgr.y (mgr.34101) 978 : cluster 0 pgmap v1572: 105 pgs: 105 active+clean; 58 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:46:35.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:34 smithi148 bash[82222]: audit 2024-03-07T03:46:33.795104+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:35.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:34 smithi195 bash[66542]: cluster 2024-03-07T03:46:33.173775+0000 mgr.y (mgr.34101) 978 : cluster 0 pgmap v1572: 105 pgs: 105 active+clean; 58 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.7 MiB/s wr, 34 op/s 2024-03-07T03:46:35.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:34 smithi195 bash[66542]: audit 2024-03-07T03:46:33.795104+0000 mon.a (mon.0) 1773 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:37.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:36 smithi148 bash[79819]: cluster 2024-03-07T03:46:35.174695+0000 mgr.y (mgr.34101) 979 : cluster 0 pgmap v1573: 105 pgs: 105 active+clean; 74 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:46:37.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:36 smithi148 bash[82222]: cluster 2024-03-07T03:46:35.174695+0000 mgr.y (mgr.34101) 979 : cluster 0 pgmap v1573: 105 pgs: 105 active+clean; 74 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:46:37.069 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:36 smithi195 bash[66542]: cluster 2024-03-07T03:46:35.174695+0000 mgr.y (mgr.34101) 979 : cluster 0 pgmap v1573: 105 pgs: 105 active+clean; 74 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 3.0 MiB/s wr, 20 op/s 2024-03-07T03:46:37.548 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:46:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:46:37] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.43.0" 2024-03-07T03:46:39.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:38 smithi148 bash[79819]: cluster 2024-03-07T03:46:37.175340+0000 mgr.y (mgr.34101) 980 : cluster 0 pgmap v1574: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-03-07T03:46:39.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:38 smithi148 bash[82222]: cluster 2024-03-07T03:46:37.175340+0000 mgr.y (mgr.34101) 980 : cluster 0 pgmap v1574: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-03-07T03:46:39.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:38 smithi195 bash[66542]: cluster 2024-03-07T03:46:37.175340+0000 mgr.y (mgr.34101) 980 : cluster 0 pgmap v1574: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-03-07T03:46:41.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:40 smithi195 bash[66542]: cluster 2024-03-07T03:46:39.176201+0000 mgr.y (mgr.34101) 981 : cluster 0 pgmap v1575: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-03-07T03:46:41.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:40 smithi148 bash[79819]: cluster 2024-03-07T03:46:39.176201+0000 mgr.y (mgr.34101) 981 : cluster 0 pgmap v1575: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-03-07T03:46:41.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:40 smithi148 bash[82222]: cluster 2024-03-07T03:46:39.176201+0000 mgr.y (mgr.34101) 981 : cluster 0 pgmap v1575: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-03-07T03:46:42.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:41 smithi195 bash[66542]: cluster 2024-03-07T03:46:40.816549+0000 mon.a (mon.0) 1774 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2024-03-07T03:46:42.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:41 smithi148 bash[82222]: cluster 2024-03-07T03:46:40.816549+0000 mon.a (mon.0) 1774 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2024-03-07T03:46:42.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:41 smithi148 bash[79819]: cluster 2024-03-07T03:46:40.816549+0000 mon.a (mon.0) 1774 : cluster 0 osdmap e976: 8 total, 8 up, 8 in 2024-03-07T03:46:42.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:46:42 smithi195 bash[59134]: ts=2024-03-07T03:46:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:43.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:42 smithi195 bash[66542]: cluster 2024-03-07T03:46:41.176781+0000 mgr.y (mgr.34101) 982 : cluster 0 pgmap v1577: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-03-07T03:46:43.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:42 smithi195 bash[66542]: audit 2024-03-07T03:46:42.347284+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:43.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:42 smithi148 bash[82222]: cluster 2024-03-07T03:46:41.176781+0000 mgr.y (mgr.34101) 982 : cluster 0 pgmap v1577: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-03-07T03:46:43.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:42 smithi148 bash[82222]: audit 2024-03-07T03:46:42.347284+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:43.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:42 smithi148 bash[79819]: cluster 2024-03-07T03:46:41.176781+0000 mgr.y (mgr.34101) 982 : cluster 0 pgmap v1577: 105 pgs: 105 active+clean; 74 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-03-07T03:46:43.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:42 smithi148 bash[79819]: audit 2024-03-07T03:46:42.347284+0000 mon.a (mon.0) 1775 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:45.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:44 smithi148 bash[79819]: cluster 2024-03-07T03:46:43.177536+0000 mgr.y (mgr.34101) 983 : cluster 0 pgmap v1578: 105 pgs: 105 active+clean; 74 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.0 MiB/s wr, 20 op/s 2024-03-07T03:46:45.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:44 smithi148 bash[82222]: cluster 2024-03-07T03:46:43.177536+0000 mgr.y (mgr.34101) 983 : cluster 0 pgmap v1578: 105 pgs: 105 active+clean; 74 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.0 MiB/s wr, 20 op/s 2024-03-07T03:46:45.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:44 smithi195 bash[66542]: cluster 2024-03-07T03:46:43.177536+0000 mgr.y (mgr.34101) 983 : cluster 0 pgmap v1578: 105 pgs: 105 active+clean; 74 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.0 MiB/s wr, 20 op/s 2024-03-07T03:46:47.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:46 smithi148 bash[79819]: cluster 2024-03-07T03:46:45.178601+0000 mgr.y (mgr.34101) 984 : cluster 0 pgmap v1579: 105 pgs: 105 active+clean; 107 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 834 KiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-03-07T03:46:47.174 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:46 smithi148 bash[82222]: cluster 2024-03-07T03:46:45.178601+0000 mgr.y (mgr.34101) 984 : cluster 0 pgmap v1579: 105 pgs: 105 active+clean; 107 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 834 KiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-03-07T03:46:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:46 smithi195 bash[66542]: cluster 2024-03-07T03:46:45.178601+0000 mgr.y (mgr.34101) 984 : cluster 0 pgmap v1579: 105 pgs: 105 active+clean; 107 MiB data, 936 MiB used, 714 GiB / 715 GiB avail; 834 KiB/s rd, 3.3 MiB/s wr, 22 op/s 2024-03-07T03:46:47.548 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:46:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:46:47] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.43.0" 2024-03-07T03:46:49.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:48 smithi148 bash[82222]: cluster 2024-03-07T03:46:47.179992+0000 mgr.y (mgr.34101) 985 : cluster 0 pgmap v1580: 105 pgs: 105 active+clean; 115 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:49.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:48 smithi148 bash[79819]: cluster 2024-03-07T03:46:47.179992+0000 mgr.y (mgr.34101) 985 : cluster 0 pgmap v1580: 105 pgs: 105 active+clean; 115 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:49.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:48 smithi195 bash[66542]: cluster 2024-03-07T03:46:47.179992+0000 mgr.y (mgr.34101) 985 : cluster 0 pgmap v1580: 105 pgs: 105 active+clean; 115 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:46:51.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:50 smithi148 bash[79819]: cluster 2024-03-07T03:46:49.181048+0000 mgr.y (mgr.34101) 986 : cluster 0 pgmap v1581: 105 pgs: 105 active+clean; 123 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.9 MiB/s wr, 39 op/s 2024-03-07T03:46:51.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:50 smithi148 bash[82222]: cluster 2024-03-07T03:46:49.181048+0000 mgr.y (mgr.34101) 986 : cluster 0 pgmap v1581: 105 pgs: 105 active+clean; 123 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.9 MiB/s wr, 39 op/s 2024-03-07T03:46:51.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:50 smithi195 bash[66542]: cluster 2024-03-07T03:46:49.181048+0000 mgr.y (mgr.34101) 986 : cluster 0 pgmap v1581: 105 pgs: 105 active+clean; 123 MiB data, 980 MiB used, 714 GiB / 715 GiB avail; 85 KiB/s rd, 4.9 MiB/s wr, 39 op/s 2024-03-07T03:46:52.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:51 smithi148 bash[79819]: audit 2024-03-07T03:46:51.554186+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:46:52.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:51 smithi148 bash[82222]: audit 2024-03-07T03:46:51.554186+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:46:52.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:51 smithi195 bash[66542]: audit 2024-03-07T03:46:51.554186+0000 mon.a (mon.0) 1776 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:46:52.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:46:52 smithi195 bash[59134]: ts=2024-03-07T03:46:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:46:53.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:52 smithi148 bash[79819]: cluster 2024-03-07T03:46:51.181653+0000 mgr.y (mgr.34101) 987 : cluster 0 pgmap v1582: 105 pgs: 105 active+clean; 123 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 4.7 MiB/s wr, 44 op/s 2024-03-07T03:46:53.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:52 smithi148 bash[82222]: cluster 2024-03-07T03:46:51.181653+0000 mgr.y (mgr.34101) 987 : cluster 0 pgmap v1582: 105 pgs: 105 active+clean; 123 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 4.7 MiB/s wr, 44 op/s 2024-03-07T03:46:53.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:52 smithi195 bash[66542]: cluster 2024-03-07T03:46:51.181653+0000 mgr.y (mgr.34101) 987 : cluster 0 pgmap v1582: 105 pgs: 105 active+clean; 123 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 186 KiB/s rd, 4.7 MiB/s wr, 44 op/s 2024-03-07T03:46:54.835 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/7.Grow (21765 ms) 2024-03-07T03:46:54.835 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/7.Shrink 2024-03-07T03:46:54.835 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:46:55.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:54 smithi148 bash[79819]: cluster 2024-03-07T03:46:53.182416+0000 mgr.y (mgr.34101) 988 : cluster 0 pgmap v1583: 105 pgs: 105 active+clean; 123 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 4.1 MiB/s wr, 38 op/s 2024-03-07T03:46:55.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:54 smithi148 bash[82222]: cluster 2024-03-07T03:46:53.182416+0000 mgr.y (mgr.34101) 988 : cluster 0 pgmap v1583: 105 pgs: 105 active+clean; 123 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 4.1 MiB/s wr, 38 op/s 2024-03-07T03:46:55.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:54 smithi195 bash[66542]: cluster 2024-03-07T03:46:53.182416+0000 mgr.y (mgr.34101) 988 : cluster 0 pgmap v1583: 105 pgs: 105 active+clean; 123 MiB data, 984 MiB used, 714 GiB / 715 GiB avail; 161 KiB/s rd, 4.1 MiB/s wr, 38 op/s 2024-03-07T03:46:57.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:56 smithi148 bash[79819]: cluster 2024-03-07T03:46:55.183621+0000 mgr.y (mgr.34101) 989 : cluster 0 pgmap v1584: 105 pgs: 105 active+clean; 143 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.1 MiB/s wr, 48 op/s 2024-03-07T03:46:57.173 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:56 smithi148 bash[82222]: cluster 2024-03-07T03:46:55.183621+0000 mgr.y (mgr.34101) 989 : cluster 0 pgmap v1584: 105 pgs: 105 active+clean; 143 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.1 MiB/s wr, 48 op/s 2024-03-07T03:46:57.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:56 smithi195 bash[66542]: cluster 2024-03-07T03:46:55.183621+0000 mgr.y (mgr.34101) 989 : cluster 0 pgmap v1584: 105 pgs: 105 active+clean; 143 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.4 MiB/s rd, 6.1 MiB/s wr, 48 op/s 2024-03-07T03:46:57.548 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:46:57] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.43.0" 2024-03-07T03:46:58.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[79819]: audit 2024-03-07T03:46:57.346906+0000 mon.a (mon.0) 1777 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:58.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[79819]: audit 2024-03-07T03:46:57.489855+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[79819]: audit 2024-03-07T03:46:57.497423+0000 mon.a (mon.0) 1779 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[79819]: audit 2024-03-07T03:46:57.593312+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[79819]: audit 2024-03-07T03:46:57.601220+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[82222]: audit 2024-03-07T03:46:57.346906+0000 mon.a (mon.0) 1777 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:58.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[82222]: audit 2024-03-07T03:46:57.489855+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[82222]: audit 2024-03-07T03:46:57.497423+0000 mon.a (mon.0) 1779 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[82222]: audit 2024-03-07T03:46:57.593312+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:57 smithi148 bash[82222]: audit 2024-03-07T03:46:57.601220+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:57 smithi195 bash[66542]: audit 2024-03-07T03:46:57.346906+0000 mon.a (mon.0) 1777 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:46:58.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:57 smithi195 bash[66542]: audit 2024-03-07T03:46:57.489855+0000 mon.a (mon.0) 1778 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:57 smithi195 bash[66542]: audit 2024-03-07T03:46:57.497423+0000 mon.a (mon.0) 1779 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:57 smithi195 bash[66542]: audit 2024-03-07T03:46:57.593312+0000 mon.a (mon.0) 1780 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:58.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:57 smithi195 bash[66542]: audit 2024-03-07T03:46:57.601220+0000 mon.a (mon.0) 1781 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:59.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[79819]: cluster 2024-03-07T03:46:57.184340+0000 mgr.y (mgr.34101) 990 : cluster 0 pgmap v1585: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.7 MiB/s wr, 44 op/s 2024-03-07T03:46:59.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[79819]: audit 2024-03-07T03:46:58.231794+0000 mon.a (mon.0) 1782 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:46:59.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[79819]: audit 2024-03-07T03:46:58.232959+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:46:59.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[79819]: audit 2024-03-07T03:46:58.242578+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:59.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[82222]: cluster 2024-03-07T03:46:57.184340+0000 mgr.y (mgr.34101) 990 : cluster 0 pgmap v1585: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.7 MiB/s wr, 44 op/s 2024-03-07T03:46:59.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[82222]: audit 2024-03-07T03:46:58.231794+0000 mon.a (mon.0) 1782 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:46:59.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[82222]: audit 2024-03-07T03:46:58.232959+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:46:59.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:58 smithi148 bash[82222]: audit 2024-03-07T03:46:58.242578+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:46:59.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:58 smithi195 bash[66542]: cluster 2024-03-07T03:46:57.184340+0000 mgr.y (mgr.34101) 990 : cluster 0 pgmap v1585: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.7 MiB/s wr, 44 op/s 2024-03-07T03:46:59.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:58 smithi195 bash[66542]: audit 2024-03-07T03:46:58.231794+0000 mon.a (mon.0) 1782 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:46:59.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:58 smithi195 bash[66542]: audit 2024-03-07T03:46:58.232959+0000 mon.a (mon.0) 1783 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:46:59.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:58 smithi195 bash[66542]: audit 2024-03-07T03:46:58.242578+0000 mon.a (mon.0) 1784 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:47:00.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:46:59 smithi148 bash[82222]: cluster 2024-03-07T03:46:59.185086+0000 mgr.y (mgr.34101) 991 : cluster 0 pgmap v1586: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:47:00.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:46:59 smithi148 bash[79819]: cluster 2024-03-07T03:46:59.185086+0000 mgr.y (mgr.34101) 991 : cluster 0 pgmap v1586: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:47:00.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:46:59 smithi195 bash[66542]: cluster 2024-03-07T03:46:59.185086+0000 mgr.y (mgr.34101) 991 : cluster 0 pgmap v1586: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:47:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:01 smithi148 bash[79819]: cluster 2024-03-07T03:47:00.882494+0000 mon.a (mon.0) 1785 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2024-03-07T03:47:02.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:01 smithi148 bash[79819]: cluster 2024-03-07T03:47:01.185814+0000 mgr.y (mgr.34101) 992 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-03-07T03:47:02.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:01 smithi148 bash[82222]: cluster 2024-03-07T03:47:00.882494+0000 mon.a (mon.0) 1785 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2024-03-07T03:47:02.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:01 smithi148 bash[82222]: cluster 2024-03-07T03:47:01.185814+0000 mgr.y (mgr.34101) 992 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-03-07T03:47:02.310 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:01 smithi195 bash[66542]: cluster 2024-03-07T03:47:00.882494+0000 mon.a (mon.0) 1785 : cluster 0 osdmap e977: 8 total, 8 up, 8 in 2024-03-07T03:47:02.311 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:01 smithi195 bash[66542]: cluster 2024-03-07T03:47:01.185814+0000 mgr.y (mgr.34101) 992 : cluster 0 pgmap v1588: 105 pgs: 105 active+clean; 147 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 2.8 MiB/s wr, 23 op/s 2024-03-07T03:47:02.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:47:02 smithi195 bash[59134]: ts=2024-03-07T03:47:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:05.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:04 smithi148 bash[79819]: cluster 2024-03-07T03:47:03.186679+0000 mgr.y (mgr.34101) 993 : cluster 0 pgmap v1589: 105 pgs: 105 active+clean; 155 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-03-07T03:47:05.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:04 smithi148 bash[82222]: cluster 2024-03-07T03:47:03.186679+0000 mgr.y (mgr.34101) 993 : cluster 0 pgmap v1589: 105 pgs: 105 active+clean; 155 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-03-07T03:47:05.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:04 smithi195 bash[66542]: cluster 2024-03-07T03:47:03.186679+0000 mgr.y (mgr.34101) 993 : cluster 0 pgmap v1589: 105 pgs: 105 active+clean; 155 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.6 MiB/s wr, 24 op/s 2024-03-07T03:47:07.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:06 smithi148 bash[79819]: cluster 2024-03-07T03:47:05.187812+0000 mgr.y (mgr.34101) 994 : cluster 0 pgmap v1590: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-03-07T03:47:07.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:06 smithi148 bash[82222]: cluster 2024-03-07T03:47:05.187812+0000 mgr.y (mgr.34101) 994 : cluster 0 pgmap v1590: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-03-07T03:47:07.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:06 smithi195 bash[66542]: cluster 2024-03-07T03:47:05.187812+0000 mgr.y (mgr.34101) 994 : cluster 0 pgmap v1590: 105 pgs: 105 active+clean; 188 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-03-07T03:47:07.548 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:47:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:47:07] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:47:09.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:08 smithi148 bash[79819]: cluster 2024-03-07T03:47:07.188459+0000 mgr.y (mgr.34101) 995 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:47:09.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:08 smithi148 bash[82222]: cluster 2024-03-07T03:47:07.188459+0000 mgr.y (mgr.34101) 995 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:47:09.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:08 smithi195 bash[66542]: cluster 2024-03-07T03:47:07.188459+0000 mgr.y (mgr.34101) 995 : cluster 0 pgmap v1591: 105 pgs: 105 active+clean; 188 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:47:11.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:10 smithi148 bash[79819]: cluster 2024-03-07T03:47:09.189486+0000 mgr.y (mgr.34101) 996 : cluster 0 pgmap v1592: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 215 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:11.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:10 smithi148 bash[82222]: cluster 2024-03-07T03:47:09.189486+0000 mgr.y (mgr.34101) 996 : cluster 0 pgmap v1592: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 215 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:11.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:10 smithi195 bash[66542]: cluster 2024-03-07T03:47:09.189486+0000 mgr.y (mgr.34101) 996 : cluster 0 pgmap v1592: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 215 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:12.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:47:12 smithi195 bash[59134]: ts=2024-03-07T03:47:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:13.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:12 smithi148 bash[79819]: cluster 2024-03-07T03:47:11.190095+0000 mgr.y (mgr.34101) 997 : cluster 0 pgmap v1593: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 4.4 MiB/s wr, 42 op/s 2024-03-07T03:47:13.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:12 smithi148 bash[79819]: audit 2024-03-07T03:47:12.347991+0000 mon.a (mon.0) 1786 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:13.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:12 smithi148 bash[82222]: cluster 2024-03-07T03:47:11.190095+0000 mgr.y (mgr.34101) 997 : cluster 0 pgmap v1593: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 4.4 MiB/s wr, 42 op/s 2024-03-07T03:47:13.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:12 smithi148 bash[82222]: audit 2024-03-07T03:47:12.347991+0000 mon.a (mon.0) 1786 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:13.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:12 smithi195 bash[66542]: cluster 2024-03-07T03:47:11.190095+0000 mgr.y (mgr.34101) 997 : cluster 0 pgmap v1593: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 4.4 MiB/s wr, 42 op/s 2024-03-07T03:47:13.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:12 smithi195 bash[66542]: audit 2024-03-07T03:47:12.347991+0000 mon.a (mon.0) 1786 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:14.649 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/7.Shrink (19814 ms) 2024-03-07T03:47:14.649 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/7.ShrinkToOne 2024-03-07T03:47:14.649 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:47:15.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:14 smithi148 bash[82222]: cluster 2024-03-07T03:47:13.190745+0000 mgr.y (mgr.34101) 998 : cluster 0 pgmap v1594: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-03-07T03:47:15.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:14 smithi148 bash[79819]: cluster 2024-03-07T03:47:13.190745+0000 mgr.y (mgr.34101) 998 : cluster 0 pgmap v1594: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-03-07T03:47:15.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:14 smithi195 bash[66542]: cluster 2024-03-07T03:47:13.190745+0000 mgr.y (mgr.34101) 998 : cluster 0 pgmap v1594: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 3.8 MiB/s wr, 36 op/s 2024-03-07T03:47:17.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:16 smithi148 bash[79819]: cluster 2024-03-07T03:47:15.191851+0000 mgr.y (mgr.34101) 999 : cluster 0 pgmap v1595: 105 pgs: 105 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 4.1 MiB/s wr, 44 op/s 2024-03-07T03:47:17.048 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:16 smithi148 bash[82222]: cluster 2024-03-07T03:47:15.191851+0000 mgr.y (mgr.34101) 999 : cluster 0 pgmap v1595: 105 pgs: 105 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 4.1 MiB/s wr, 44 op/s 2024-03-07T03:47:17.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:16 smithi195 bash[66542]: cluster 2024-03-07T03:47:15.191851+0000 mgr.y (mgr.34101) 999 : cluster 0 pgmap v1595: 105 pgs: 105 active+clean; 200 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 4.1 MiB/s wr, 44 op/s 2024-03-07T03:47:17.549 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:47:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:47:17] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-03-07T03:47:19.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:18 smithi148 bash[79819]: cluster 2024-03-07T03:47:17.192573+0000 mgr.y (mgr.34101) 1000 : cluster 0 pgmap v1596: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.0 MiB/s wr, 30 op/s 2024-03-07T03:47:19.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:18 smithi148 bash[82222]: cluster 2024-03-07T03:47:17.192573+0000 mgr.y (mgr.34101) 1000 : cluster 0 pgmap v1596: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.0 MiB/s wr, 30 op/s 2024-03-07T03:47:19.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:18 smithi195 bash[66542]: cluster 2024-03-07T03:47:17.192573+0000 mgr.y (mgr.34101) 1000 : cluster 0 pgmap v1596: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 3.0 MiB/s wr, 30 op/s 2024-03-07T03:47:21.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:20 smithi148 bash[79819]: cluster 2024-03-07T03:47:19.193333+0000 mgr.y (mgr.34101) 1001 : cluster 0 pgmap v1597: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:47:21.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:20 smithi148 bash[82222]: cluster 2024-03-07T03:47:19.193333+0000 mgr.y (mgr.34101) 1001 : cluster 0 pgmap v1597: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:47:21.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:20 smithi195 bash[66542]: cluster 2024-03-07T03:47:19.193333+0000 mgr.y (mgr.34101) 1001 : cluster 0 pgmap v1597: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-03-07T03:47:22.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:47:22 smithi195 bash[59134]: ts=2024-03-07T03:47:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:23.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:22 smithi148 bash[79819]: cluster 2024-03-07T03:47:21.193943+0000 mgr.y (mgr.34101) 1002 : cluster 0 pgmap v1598: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:47:23.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:22 smithi148 bash[82222]: cluster 2024-03-07T03:47:21.193943+0000 mgr.y (mgr.34101) 1002 : cluster 0 pgmap v1598: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:47:23.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:22 smithi195 bash[66542]: cluster 2024-03-07T03:47:21.193943+0000 mgr.y (mgr.34101) 1002 : cluster 0 pgmap v1598: 105 pgs: 105 active+clean; 220 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.6 MiB/s rd, 2.7 MiB/s wr, 27 op/s 2024-03-07T03:47:24.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:23 smithi148 bash[79819]: cluster 2024-03-07T03:47:22.784887+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2024-03-07T03:47:24.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:23 smithi148 bash[82222]: cluster 2024-03-07T03:47:22.784887+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2024-03-07T03:47:24.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:23 smithi195 bash[66542]: cluster 2024-03-07T03:47:22.784887+0000 mon.a (mon.0) 1787 : cluster 0 osdmap e978: 8 total, 8 up, 8 in 2024-03-07T03:47:25.048 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:24 smithi148 bash[79819]: cluster 2024-03-07T03:47:23.194433+0000 mgr.y (mgr.34101) 1003 : cluster 0 pgmap v1600: 105 pgs: 105 active+clean; 221 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-03-07T03:47:25.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:24 smithi148 bash[82222]: cluster 2024-03-07T03:47:23.194433+0000 mgr.y (mgr.34101) 1003 : cluster 0 pgmap v1600: 105 pgs: 105 active+clean; 221 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-03-07T03:47:25.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:24 smithi195 bash[66542]: cluster 2024-03-07T03:47:23.194433+0000 mgr.y (mgr.34101) 1003 : cluster 0 pgmap v1600: 105 pgs: 105 active+clean; 221 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 27 op/s 2024-03-07T03:47:27.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:26 smithi195 bash[66542]: cluster 2024-03-07T03:47:25.195040+0000 mgr.y (mgr.34101) 1004 : cluster 0 pgmap v1601: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:47:27.173 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:26 smithi148 bash[79819]: cluster 2024-03-07T03:47:25.195040+0000 mgr.y (mgr.34101) 1004 : cluster 0 pgmap v1601: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:47:27.174 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:26 smithi148 bash[82222]: cluster 2024-03-07T03:47:25.195040+0000 mgr.y (mgr.34101) 1004 : cluster 0 pgmap v1601: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 6.1 MiB/s rd, 4.5 MiB/s wr, 39 op/s 2024-03-07T03:47:27.548 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:47:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:47:27] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-03-07T03:47:28.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:27 smithi195 bash[66542]: audit 2024-03-07T03:47:27.347677+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:28.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:27 smithi148 bash[82222]: audit 2024-03-07T03:47:27.347677+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:28.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:27 smithi148 bash[79819]: audit 2024-03-07T03:47:27.347677+0000 mon.a (mon.0) 1788 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:29.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:28 smithi195 bash[66542]: cluster 2024-03-07T03:47:27.195872+0000 mgr.y (mgr.34101) 1005 : cluster 0 pgmap v1602: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 540 KiB/s rd, 4.1 MiB/s wr, 38 op/s 2024-03-07T03:47:29.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:28 smithi148 bash[79819]: cluster 2024-03-07T03:47:27.195872+0000 mgr.y (mgr.34101) 1005 : cluster 0 pgmap v1602: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 540 KiB/s rd, 4.1 MiB/s wr, 38 op/s 2024-03-07T03:47:29.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:28 smithi148 bash[82222]: cluster 2024-03-07T03:47:27.195872+0000 mgr.y (mgr.34101) 1005 : cluster 0 pgmap v1602: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 540 KiB/s rd, 4.1 MiB/s wr, 38 op/s 2024-03-07T03:47:31.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:30 smithi195 bash[66542]: cluster 2024-03-07T03:47:29.196693+0000 mgr.y (mgr.34101) 1006 : cluster 0 pgmap v1603: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:31.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:30 smithi148 bash[79819]: cluster 2024-03-07T03:47:29.196693+0000 mgr.y (mgr.34101) 1006 : cluster 0 pgmap v1603: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:31.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:30 smithi148 bash[82222]: cluster 2024-03-07T03:47:29.196693+0000 mgr.y (mgr.34101) 1006 : cluster 0 pgmap v1603: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:32.570 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:47:32 smithi195 bash[59134]: ts=2024-03-07T03:47:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:33.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:32 smithi148 bash[79819]: cluster 2024-03-07T03:47:31.197482+0000 mgr.y (mgr.34101) 1007 : cluster 0 pgmap v1604: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:33.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:32 smithi148 bash[82222]: cluster 2024-03-07T03:47:31.197482+0000 mgr.y (mgr.34101) 1007 : cluster 0 pgmap v1604: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:33.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:32 smithi195 bash[66542]: cluster 2024-03-07T03:47:31.197482+0000 mgr.y (mgr.34101) 1007 : cluster 0 pgmap v1604: 105 pgs: 105 active+clean; 265 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:47:35.298 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:34 smithi148 bash[82222]: cluster 2024-03-07T03:47:33.198141+0000 mgr.y (mgr.34101) 1008 : cluster 0 pgmap v1605: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 4.7 MiB/s wr, 43 op/s 2024-03-07T03:47:35.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:34 smithi148 bash[79819]: cluster 2024-03-07T03:47:33.198141+0000 mgr.y (mgr.34101) 1008 : cluster 0 pgmap v1605: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 4.7 MiB/s wr, 43 op/s 2024-03-07T03:47:35.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:34 smithi195 bash[66542]: cluster 2024-03-07T03:47:33.198141+0000 mgr.y (mgr.34101) 1008 : cluster 0 pgmap v1605: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 4.7 MiB/s wr, 43 op/s 2024-03-07T03:47:37.058 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/7.ShrinkToOne (22409 ms) 2024-03-07T03:47:37.058 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/7.ShrinkToOneAfterSnapshot 2024-03-07T03:47:37.059 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:47:37.174 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:36 smithi148 bash[79819]: cluster 2024-03-07T03:47:35.199072+0000 mgr.y (mgr.34101) 1009 : cluster 0 pgmap v1606: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 4.0 MiB/s wr, 37 op/s 2024-03-07T03:47:37.175 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:36 smithi148 bash[82222]: cluster 2024-03-07T03:47:35.199072+0000 mgr.y (mgr.34101) 1009 : cluster 0 pgmap v1606: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 4.0 MiB/s wr, 37 op/s 2024-03-07T03:47:37.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:36 smithi195 bash[66542]: cluster 2024-03-07T03:47:35.199072+0000 mgr.y (mgr.34101) 1009 : cluster 0 pgmap v1606: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 4.0 MiB/s wr, 37 op/s 2024-03-07T03:47:37.548 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:47:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:47:37] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.43.0" 2024-03-07T03:47:39.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:38 smithi148 bash[79819]: cluster 2024-03-07T03:47:37.200076+0000 mgr.y (mgr.34101) 1010 : cluster 0 pgmap v1607: 105 pgs: 105 active+clean; 273 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 534 KiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-03-07T03:47:39.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:38 smithi148 bash[82222]: cluster 2024-03-07T03:47:37.200076+0000 mgr.y (mgr.34101) 1010 : cluster 0 pgmap v1607: 105 pgs: 105 active+clean; 273 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 534 KiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-03-07T03:47:39.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:38 smithi195 bash[66542]: cluster 2024-03-07T03:47:37.200076+0000 mgr.y (mgr.34101) 1010 : cluster 0 pgmap v1607: 105 pgs: 105 active+clean; 273 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 534 KiB/s rd, 2.7 MiB/s wr, 19 op/s 2024-03-07T03:47:41.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:40 smithi148 bash[79819]: cluster 2024-03-07T03:47:39.201001+0000 mgr.y (mgr.34101) 1011 : cluster 0 pgmap v1608: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 903 KiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-03-07T03:47:41.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:40 smithi148 bash[82222]: cluster 2024-03-07T03:47:39.201001+0000 mgr.y (mgr.34101) 1011 : cluster 0 pgmap v1608: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 903 KiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-03-07T03:47:41.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:40 smithi195 bash[66542]: cluster 2024-03-07T03:47:39.201001+0000 mgr.y (mgr.34101) 1011 : cluster 0 pgmap v1608: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 903 KiB/s rd, 1.7 MiB/s wr, 22 op/s 2024-03-07T03:47:42.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:47:42 smithi195 bash[59134]: ts=2024-03-07T03:47:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:43.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:42 smithi148 bash[79819]: cluster 2024-03-07T03:47:41.201576+0000 mgr.y (mgr.34101) 1012 : cluster 0 pgmap v1609: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-03-07T03:47:43.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:42 smithi148 bash[79819]: audit 2024-03-07T03:47:42.348126+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:43.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:42 smithi148 bash[82222]: cluster 2024-03-07T03:47:41.201576+0000 mgr.y (mgr.34101) 1012 : cluster 0 pgmap v1609: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-03-07T03:47:43.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:42 smithi148 bash[82222]: audit 2024-03-07T03:47:42.348126+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:43.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:42 smithi195 bash[66542]: cluster 2024-03-07T03:47:41.201576+0000 mgr.y (mgr.34101) 1012 : cluster 0 pgmap v1609: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-03-07T03:47:43.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:42 smithi195 bash[66542]: audit 2024-03-07T03:47:42.348126+0000 mon.a (mon.0) 1789 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:45.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:44 smithi148 bash[82222]: cluster 2024-03-07T03:47:43.202300+0000 mgr.y (mgr.34101) 1013 : cluster 0 pgmap v1610: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-03-07T03:47:45.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:44 smithi148 bash[79819]: cluster 2024-03-07T03:47:43.202300+0000 mgr.y (mgr.34101) 1013 : cluster 0 pgmap v1610: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-03-07T03:47:45.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:44 smithi195 bash[66542]: cluster 2024-03-07T03:47:43.202300+0000 mgr.y (mgr.34101) 1013 : cluster 0 pgmap v1610: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 1.3 MiB/s wr, 22 op/s 2024-03-07T03:47:46.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:45 smithi148 bash[79819]: cluster 2024-03-07T03:47:44.864800+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2024-03-07T03:47:46.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:45 smithi148 bash[82222]: cluster 2024-03-07T03:47:44.864800+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2024-03-07T03:47:46.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:45 smithi195 bash[66542]: cluster 2024-03-07T03:47:44.864800+0000 mon.a (mon.0) 1790 : cluster 0 osdmap e979: 8 total, 8 up, 8 in 2024-03-07T03:47:47.174 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:46 smithi148 bash[79819]: cluster 2024-03-07T03:47:45.202956+0000 mgr.y (mgr.34101) 1014 : cluster 0 pgmap v1612: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 889 KiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-03-07T03:47:47.174 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:46 smithi148 bash[82222]: cluster 2024-03-07T03:47:45.202956+0000 mgr.y (mgr.34101) 1014 : cluster 0 pgmap v1612: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 889 KiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-03-07T03:47:47.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:46 smithi195 bash[66542]: cluster 2024-03-07T03:47:45.202956+0000 mgr.y (mgr.34101) 1014 : cluster 0 pgmap v1612: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 889 KiB/s rd, 2.0 MiB/s wr, 18 op/s 2024-03-07T03:47:47.549 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:47:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:47:47] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-03-07T03:47:48.298 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:47 smithi148 bash[79819]: cluster 2024-03-07T03:47:47.203816+0000 mgr.y (mgr.34101) 1015 : cluster 0 pgmap v1613: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 456 KiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-03-07T03:47:48.299 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:47 smithi148 bash[82222]: cluster 2024-03-07T03:47:47.203816+0000 mgr.y (mgr.34101) 1015 : cluster 0 pgmap v1613: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 456 KiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-03-07T03:47:48.320 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:47 smithi195 bash[66542]: cluster 2024-03-07T03:47:47.203816+0000 mgr.y (mgr.34101) 1015 : cluster 0 pgmap v1613: 105 pgs: 105 active+clean; 289 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 456 KiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-03-07T03:47:50.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:50 smithi148 bash[79819]: cluster 2024-03-07T03:47:49.204546+0000 mgr.y (mgr.34101) 1016 : cluster 0 pgmap v1614: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:47:50.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:50 smithi148 bash[82222]: cluster 2024-03-07T03:47:49.204546+0000 mgr.y (mgr.34101) 1016 : cluster 0 pgmap v1614: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:47:50.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:50 smithi195 bash[66542]: cluster 2024-03-07T03:47:49.204546+0000 mgr.y (mgr.34101) 1016 : cluster 0 pgmap v1614: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:47:52.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:52 smithi148 bash[79819]: cluster 2024-03-07T03:47:51.205129+0000 mgr.y (mgr.34101) 1017 : cluster 0 pgmap v1615: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:47:52.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:52 smithi148 bash[82222]: cluster 2024-03-07T03:47:51.205129+0000 mgr.y (mgr.34101) 1017 : cluster 0 pgmap v1615: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:47:52.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:52 smithi195 bash[66542]: cluster 2024-03-07T03:47:51.205129+0000 mgr.y (mgr.34101) 1017 : cluster 0 pgmap v1615: 105 pgs: 105 active+clean; 310 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-03-07T03:47:52.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:47:52 smithi195 bash[59134]: ts=2024-03-07T03:47:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:47:54.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:54 smithi148 bash[82222]: cluster 2024-03-07T03:47:53.205680+0000 mgr.y (mgr.34101) 1018 : cluster 0 pgmap v1616: 105 pgs: 105 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:47:54.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:54 smithi148 bash[79819]: cluster 2024-03-07T03:47:53.205680+0000 mgr.y (mgr.34101) 1018 : cluster 0 pgmap v1616: 105 pgs: 105 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:47:54.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:54 smithi195 bash[66542]: cluster 2024-03-07T03:47:53.205680+0000 mgr.y (mgr.34101) 1018 : cluster 0 pgmap v1616: 105 pgs: 105 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 118 KiB/s rd, 4.5 MiB/s wr, 43 op/s 2024-03-07T03:47:56.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:56 smithi148 bash[79819]: cluster 2024-03-07T03:47:55.206314+0000 mgr.y (mgr.34101) 1019 : cluster 0 pgmap v1617: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 4.7 MiB/s wr, 43 op/s 2024-03-07T03:47:56.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:56 smithi148 bash[82222]: cluster 2024-03-07T03:47:55.206314+0000 mgr.y (mgr.34101) 1019 : cluster 0 pgmap v1617: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 4.7 MiB/s wr, 43 op/s 2024-03-07T03:47:56.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:56 smithi195 bash[66542]: cluster 2024-03-07T03:47:55.206314+0000 mgr.y (mgr.34101) 1019 : cluster 0 pgmap v1617: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 213 KiB/s rd, 4.7 MiB/s wr, 43 op/s 2024-03-07T03:47:57.549 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:47:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:47:57] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.43.0" 2024-03-07T03:47:58.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:58 smithi148 bash[79819]: cluster 2024-03-07T03:47:57.207241+0000 mgr.y (mgr.34101) 1020 : cluster 0 pgmap v1618: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 185 KiB/s rd, 3.4 MiB/s wr, 38 op/s 2024-03-07T03:47:58.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:58 smithi148 bash[79819]: cluster 2024-03-07T03:47:57.285763+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2024-03-07T03:47:58.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:58 smithi148 bash[79819]: audit 2024-03-07T03:47:57.350156+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:58.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:58 smithi148 bash[82222]: cluster 2024-03-07T03:47:57.207241+0000 mgr.y (mgr.34101) 1020 : cluster 0 pgmap v1618: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 185 KiB/s rd, 3.4 MiB/s wr, 38 op/s 2024-03-07T03:47:58.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:58 smithi148 bash[82222]: cluster 2024-03-07T03:47:57.285763+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2024-03-07T03:47:58.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:58 smithi148 bash[82222]: audit 2024-03-07T03:47:57.350156+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:58.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:58 smithi195 bash[66542]: cluster 2024-03-07T03:47:57.207241+0000 mgr.y (mgr.34101) 1020 : cluster 0 pgmap v1618: 105 pgs: 105 active+clean; 318 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 185 KiB/s rd, 3.4 MiB/s wr, 38 op/s 2024-03-07T03:47:58.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:58 smithi195 bash[66542]: cluster 2024-03-07T03:47:57.285763+0000 mon.a (mon.0) 1791 : cluster 0 osdmap e980: 8 total, 8 up, 8 in 2024-03-07T03:47:58.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:58 smithi195 bash[66542]: audit 2024-03-07T03:47:57.350156+0000 mon.a (mon.0) 1792 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:47:59.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:47:59 smithi148 bash[82222]: audit 2024-03-07T03:47:58.322983+0000 mon.a (mon.0) 1793 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:47:59.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:47:59 smithi148 bash[79819]: audit 2024-03-07T03:47:58.322983+0000 mon.a (mon.0) 1793 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:47:59.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:47:59 smithi195 bash[66542]: audit 2024-03-07T03:47:58.322983+0000 mon.a (mon.0) 1793 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:47:59.801 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/7.ShrinkToOneAfterSnapshot (22742 ms) 2024-03-07T03:47:59.801 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/7.MinOverlap 2024-03-07T03:47:59.801 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:48:00.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:00 smithi148 bash[79819]: cluster 2024-03-07T03:47:59.208369+0000 mgr.y (mgr.34101) 1021 : cluster 0 pgmap v1620: 105 pgs: 105 active+clean; 358 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:48:00.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:00 smithi148 bash[82222]: cluster 2024-03-07T03:47:59.208369+0000 mgr.y (mgr.34101) 1021 : cluster 0 pgmap v1620: 105 pgs: 105 active+clean; 358 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:48:00.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:00 smithi195 bash[66542]: cluster 2024-03-07T03:47:59.208369+0000 mgr.y (mgr.34101) 1021 : cluster 0 pgmap v1620: 105 pgs: 105 active+clean; 358 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-03-07T03:48:02.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:02 smithi148 bash[79819]: cluster 2024-03-07T03:48:01.208984+0000 mgr.y (mgr.34101) 1022 : cluster 0 pgmap v1621: 105 pgs: 105 active+clean; 366 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 5.6 MiB/s wr, 31 op/s 2024-03-07T03:48:02.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:02 smithi148 bash[82222]: cluster 2024-03-07T03:48:01.208984+0000 mgr.y (mgr.34101) 1022 : cluster 0 pgmap v1621: 105 pgs: 105 active+clean; 366 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 5.6 MiB/s wr, 31 op/s 2024-03-07T03:48:02.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:02 smithi195 bash[66542]: cluster 2024-03-07T03:48:01.208984+0000 mgr.y (mgr.34101) 1022 : cluster 0 pgmap v1621: 105 pgs: 105 active+clean; 366 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 5.6 MiB/s wr, 31 op/s 2024-03-07T03:48:02.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:48:02 smithi195 bash[59134]: ts=2024-03-07T03:48:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:04.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:04 smithi148 bash[79819]: cluster 2024-03-07T03:48:03.209680+0000 mgr.y (mgr.34101) 1023 : cluster 0 pgmap v1622: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 36 op/s 2024-03-07T03:48:04.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:04 smithi148 bash[79819]: audit 2024-03-07T03:48:04.260533+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:04.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:04 smithi148 bash[79819]: audit 2024-03-07T03:48:04.268057+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:04.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:04 smithi148 bash[82222]: cluster 2024-03-07T03:48:03.209680+0000 mgr.y (mgr.34101) 1023 : cluster 0 pgmap v1622: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 36 op/s 2024-03-07T03:48:04.549 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:04 smithi148 bash[82222]: audit 2024-03-07T03:48:04.260533+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:04.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:04 smithi148 bash[82222]: audit 2024-03-07T03:48:04.268057+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:04.570 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:04 smithi195 bash[66542]: cluster 2024-03-07T03:48:03.209680+0000 mgr.y (mgr.34101) 1023 : cluster 0 pgmap v1622: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 36 op/s 2024-03-07T03:48:04.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:04 smithi195 bash[66542]: audit 2024-03-07T03:48:04.260533+0000 mon.a (mon.0) 1794 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:04.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:04 smithi195 bash[66542]: audit 2024-03-07T03:48:04.268057+0000 mon.a (mon.0) 1795 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[79819]: audit 2024-03-07T03:48:04.409717+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[79819]: audit 2024-03-07T03:48:04.417741+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[79819]: audit 2024-03-07T03:48:05.007054+0000 mon.a (mon.0) 1798 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:48:05.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[79819]: audit 2024-03-07T03:48:05.008098+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:48:05.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[79819]: audit 2024-03-07T03:48:05.014031+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[82222]: audit 2024-03-07T03:48:04.409717+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[82222]: audit 2024-03-07T03:48:04.417741+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[82222]: audit 2024-03-07T03:48:05.007054+0000 mon.a (mon.0) 1798 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:48:05.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[82222]: audit 2024-03-07T03:48:05.008098+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:48:05.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:05 smithi148 bash[82222]: audit 2024-03-07T03:48:05.014031+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:05 smithi195 bash[66542]: audit 2024-03-07T03:48:04.409717+0000 mon.a (mon.0) 1796 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:05 smithi195 bash[66542]: audit 2024-03-07T03:48:04.417741+0000 mon.a (mon.0) 1797 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:05.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:05 smithi195 bash[66542]: audit 2024-03-07T03:48:05.007054+0000 mon.a (mon.0) 1798 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:48:05.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:05 smithi195 bash[66542]: audit 2024-03-07T03:48:05.008098+0000 mon.a (mon.0) 1799 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:48:05.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:05 smithi195 bash[66542]: audit 2024-03-07T03:48:05.014031+0000 mon.a (mon.0) 1800 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:48:06.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:06 smithi148 bash[79819]: cluster 2024-03-07T03:48:05.210347+0000 mgr.y (mgr.34101) 1024 : cluster 0 pgmap v1623: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 36 op/s 2024-03-07T03:48:06.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:06 smithi148 bash[82222]: cluster 2024-03-07T03:48:05.210347+0000 mgr.y (mgr.34101) 1024 : cluster 0 pgmap v1623: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 36 op/s 2024-03-07T03:48:06.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:06 smithi195 bash[66542]: cluster 2024-03-07T03:48:05.210347+0000 mgr.y (mgr.34101) 1024 : cluster 0 pgmap v1623: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.6 MiB/s wr, 36 op/s 2024-03-07T03:48:07.433 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:48:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:48:07] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.43.0" 2024-03-07T03:48:07.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:07 smithi148 bash[79819]: cluster 2024-03-07T03:48:06.423639+0000 mon.a (mon.0) 1801 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2024-03-07T03:48:07.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:07 smithi148 bash[82222]: cluster 2024-03-07T03:48:06.423639+0000 mon.a (mon.0) 1801 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2024-03-07T03:48:07.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:07 smithi195 bash[66542]: cluster 2024-03-07T03:48:06.423639+0000 mon.a (mon.0) 1801 : cluster 0 osdmap e981: 8 total, 8 up, 8 in 2024-03-07T03:48:08.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:08 smithi148 bash[79819]: cluster 2024-03-07T03:48:07.210857+0000 mgr.y (mgr.34101) 1025 : cluster 0 pgmap v1625: 105 pgs: 105 active+clean; 378 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 6.0 MiB/s wr, 35 op/s 2024-03-07T03:48:08.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:08 smithi148 bash[82222]: cluster 2024-03-07T03:48:07.210857+0000 mgr.y (mgr.34101) 1025 : cluster 0 pgmap v1625: 105 pgs: 105 active+clean; 378 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 6.0 MiB/s wr, 35 op/s 2024-03-07T03:48:08.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:08 smithi195 bash[66542]: cluster 2024-03-07T03:48:07.210857+0000 mgr.y (mgr.34101) 1025 : cluster 0 pgmap v1625: 105 pgs: 105 active+clean; 378 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 6.0 MiB/s wr, 35 op/s 2024-03-07T03:48:10.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:10 smithi148 bash[79819]: cluster 2024-03-07T03:48:09.211446+0000 mgr.y (mgr.34101) 1026 : cluster 0 pgmap v1626: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-03-07T03:48:10.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:10 smithi148 bash[82222]: cluster 2024-03-07T03:48:09.211446+0000 mgr.y (mgr.34101) 1026 : cluster 0 pgmap v1626: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-03-07T03:48:10.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:10 smithi195 bash[66542]: cluster 2024-03-07T03:48:09.211446+0000 mgr.y (mgr.34101) 1026 : cluster 0 pgmap v1626: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-03-07T03:48:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:12 smithi195 bash[66542]: cluster 2024-03-07T03:48:11.212241+0000 mgr.y (mgr.34101) 1027 : cluster 0 pgmap v1627: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 4.9 MiB/s wr, 48 op/s 2024-03-07T03:48:12.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:12 smithi195 bash[66542]: audit 2024-03-07T03:48:12.349731+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:12.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:48:12 smithi195 bash[59134]: ts=2024-03-07T03:48:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:12.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:12 smithi148 bash[79819]: cluster 2024-03-07T03:48:11.212241+0000 mgr.y (mgr.34101) 1027 : cluster 0 pgmap v1627: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 4.9 MiB/s wr, 48 op/s 2024-03-07T03:48:12.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:12 smithi148 bash[79819]: audit 2024-03-07T03:48:12.349731+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:12.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:12 smithi148 bash[82222]: cluster 2024-03-07T03:48:11.212241+0000 mgr.y (mgr.34101) 1027 : cluster 0 pgmap v1627: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 34 KiB/s rd, 4.9 MiB/s wr, 48 op/s 2024-03-07T03:48:12.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:12 smithi148 bash[82222]: audit 2024-03-07T03:48:12.349731+0000 mon.a (mon.0) 1802 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:14.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:14 smithi148 bash[79819]: cluster 2024-03-07T03:48:13.212998+0000 mgr.y (mgr.34101) 1028 : cluster 0 pgmap v1628: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:48:14.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:14 smithi148 bash[82222]: cluster 2024-03-07T03:48:13.212998+0000 mgr.y (mgr.34101) 1028 : cluster 0 pgmap v1628: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:48:14.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:14 smithi195 bash[66542]: cluster 2024-03-07T03:48:13.212998+0000 mgr.y (mgr.34101) 1028 : cluster 0 pgmap v1628: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 27 KiB/s rd, 4.1 MiB/s wr, 37 op/s 2024-03-07T03:48:16.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:16 smithi148 bash[79819]: cluster 2024-03-07T03:48:15.213865+0000 mgr.y (mgr.34101) 1029 : cluster 0 pgmap v1629: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:48:16.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:16 smithi148 bash[82222]: cluster 2024-03-07T03:48:15.213865+0000 mgr.y (mgr.34101) 1029 : cluster 0 pgmap v1629: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:48:16.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:16 smithi195 bash[66542]: cluster 2024-03-07T03:48:15.213865+0000 mgr.y (mgr.34101) 1029 : cluster 0 pgmap v1629: 105 pgs: 105 active+clean; 415 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-03-07T03:48:17.549 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:48:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:48:17] "GET /metrics HTTP/1.1" 200 36748 "" "Prometheus/2.43.0" 2024-03-07T03:48:18.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:18 smithi148 bash[79819]: cluster 2024-03-07T03:48:17.214500+0000 mgr.y (mgr.34101) 1030 : cluster 0 pgmap v1630: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 131 KiB/s rd, 4.2 MiB/s wr, 34 op/s 2024-03-07T03:48:18.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:18 smithi148 bash[82222]: cluster 2024-03-07T03:48:17.214500+0000 mgr.y (mgr.34101) 1030 : cluster 0 pgmap v1630: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 131 KiB/s rd, 4.2 MiB/s wr, 34 op/s 2024-03-07T03:48:18.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:18 smithi195 bash[66542]: cluster 2024-03-07T03:48:17.214500+0000 mgr.y (mgr.34101) 1030 : cluster 0 pgmap v1630: 105 pgs: 105 active+clean; 419 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 131 KiB/s rd, 4.2 MiB/s wr, 34 op/s 2024-03-07T03:48:20.331 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/7.MinOverlap (20531 ms) 2024-03-07T03:48:20.332 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] EncryptedFlattenTest/7.ZeroOverlap 2024-03-07T03:48:20.332 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:48:20.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:20 smithi148 bash[79819]: cluster 2024-03-07T03:48:19.215552+0000 mgr.y (mgr.34101) 1031 : cluster 0 pgmap v1631: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.4 MiB/s wr, 50 op/s 2024-03-07T03:48:20.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:20 smithi148 bash[82222]: cluster 2024-03-07T03:48:19.215552+0000 mgr.y (mgr.34101) 1031 : cluster 0 pgmap v1631: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.4 MiB/s wr, 50 op/s 2024-03-07T03:48:20.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:20 smithi195 bash[66542]: cluster 2024-03-07T03:48:19.215552+0000 mgr.y (mgr.34101) 1031 : cluster 0 pgmap v1631: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.4 MiB/s wr, 50 op/s 2024-03-07T03:48:22.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:22 smithi195 bash[66542]: cluster 2024-03-07T03:48:21.216407+0000 mgr.y (mgr.34101) 1032 : cluster 0 pgmap v1632: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-03-07T03:48:22.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:48:22 smithi195 bash[59134]: ts=2024-03-07T03:48:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:22.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:22 smithi148 bash[79819]: cluster 2024-03-07T03:48:21.216407+0000 mgr.y (mgr.34101) 1032 : cluster 0 pgmap v1632: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-03-07T03:48:22.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:22 smithi148 bash[82222]: cluster 2024-03-07T03:48:21.216407+0000 mgr.y (mgr.34101) 1032 : cluster 0 pgmap v1632: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 3.0 MiB/s wr, 22 op/s 2024-03-07T03:48:24.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:24 smithi148 bash[79819]: cluster 2024-03-07T03:48:23.217165+0000 mgr.y (mgr.34101) 1033 : cluster 0 pgmap v1633: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-03-07T03:48:24.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:24 smithi148 bash[82222]: cluster 2024-03-07T03:48:23.217165+0000 mgr.y (mgr.34101) 1033 : cluster 0 pgmap v1633: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-03-07T03:48:24.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:24 smithi195 bash[66542]: cluster 2024-03-07T03:48:23.217165+0000 mgr.y (mgr.34101) 1033 : cluster 0 pgmap v1633: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.5 MiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-03-07T03:48:26.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:26 smithi148 bash[79819]: cluster 2024-03-07T03:48:25.217819+0000 mgr.y (mgr.34101) 1034 : cluster 0 pgmap v1634: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-03-07T03:48:26.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:26 smithi148 bash[82222]: cluster 2024-03-07T03:48:25.217819+0000 mgr.y (mgr.34101) 1034 : cluster 0 pgmap v1634: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-03-07T03:48:26.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:26 smithi195 bash[66542]: cluster 2024-03-07T03:48:25.217819+0000 mgr.y (mgr.34101) 1034 : cluster 0 pgmap v1634: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-03-07T03:48:27.476 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:48:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:48:27] "GET /metrics HTTP/1.1" 200 36748 "" "Prometheus/2.43.0" 2024-03-07T03:48:27.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:27 smithi148 bash[79819]: audit 2024-03-07T03:48:27.349518+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:27.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:27 smithi148 bash[82222]: audit 2024-03-07T03:48:27.349518+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:27.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:27 smithi195 bash[66542]: audit 2024-03-07T03:48:27.349518+0000 mon.a (mon.0) 1803 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:28.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:28 smithi148 bash[79819]: cluster 2024-03-07T03:48:27.218437+0000 mgr.y (mgr.34101) 1035 : cluster 0 pgmap v1635: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-03-07T03:48:28.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:28 smithi148 bash[82222]: cluster 2024-03-07T03:48:27.218437+0000 mgr.y (mgr.34101) 1035 : cluster 0 pgmap v1635: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-03-07T03:48:28.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:28 smithi195 bash[66542]: cluster 2024-03-07T03:48:27.218437+0000 mgr.y (mgr.34101) 1035 : cluster 0 pgmap v1635: 105 pgs: 105 active+clean; 423 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.7 MiB/s wr, 24 op/s 2024-03-07T03:48:29.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:29 smithi148 bash[79819]: cluster 2024-03-07T03:48:28.497554+0000 mon.a (mon.0) 1804 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2024-03-07T03:48:29.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:29 smithi148 bash[82222]: cluster 2024-03-07T03:48:28.497554+0000 mon.a (mon.0) 1804 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2024-03-07T03:48:29.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:29 smithi195 bash[66542]: cluster 2024-03-07T03:48:28.497554+0000 mon.a (mon.0) 1804 : cluster 0 osdmap e982: 8 total, 8 up, 8 in 2024-03-07T03:48:30.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:30 smithi148 bash[82222]: cluster 2024-03-07T03:48:29.218977+0000 mgr.y (mgr.34101) 1036 : cluster 0 pgmap v1637: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 857 KiB/s rd, 3.3 MiB/s wr, 15 op/s 2024-03-07T03:48:30.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:30 smithi148 bash[79819]: cluster 2024-03-07T03:48:29.218977+0000 mgr.y (mgr.34101) 1036 : cluster 0 pgmap v1637: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 857 KiB/s rd, 3.3 MiB/s wr, 15 op/s 2024-03-07T03:48:30.820 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:30 smithi195 bash[66542]: cluster 2024-03-07T03:48:29.218977+0000 mgr.y (mgr.34101) 1036 : cluster 0 pgmap v1637: 105 pgs: 105 active+clean; 452 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 857 KiB/s rd, 3.3 MiB/s wr, 15 op/s 2024-03-07T03:48:32.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:32 smithi195 bash[66542]: cluster 2024-03-07T03:48:31.219437+0000 mgr.y (mgr.34101) 1037 : cluster 0 pgmap v1638: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 432 KiB/s rd, 3.3 MiB/s wr, 32 op/s 2024-03-07T03:48:32.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:48:32 smithi195 bash[59134]: ts=2024-03-07T03:48:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:32.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:32 smithi148 bash[79819]: cluster 2024-03-07T03:48:31.219437+0000 mgr.y (mgr.34101) 1037 : cluster 0 pgmap v1638: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 432 KiB/s rd, 3.3 MiB/s wr, 32 op/s 2024-03-07T03:48:32.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:32 smithi148 bash[82222]: cluster 2024-03-07T03:48:31.219437+0000 mgr.y (mgr.34101) 1037 : cluster 0 pgmap v1638: 105 pgs: 105 active+clean; 456 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 432 KiB/s rd, 3.3 MiB/s wr, 32 op/s 2024-03-07T03:48:34.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:34 smithi148 bash[79819]: cluster 2024-03-07T03:48:33.219983+0000 mgr.y (mgr.34101) 1038 : cluster 0 pgmap v1639: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 438 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:48:34.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:34 smithi148 bash[82222]: cluster 2024-03-07T03:48:33.219983+0000 mgr.y (mgr.34101) 1038 : cluster 0 pgmap v1639: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 438 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:48:34.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:34 smithi195 bash[66542]: cluster 2024-03-07T03:48:33.219983+0000 mgr.y (mgr.34101) 1038 : cluster 0 pgmap v1639: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 438 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-03-07T03:48:36.799 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:36 smithi148 bash[82222]: cluster 2024-03-07T03:48:35.220617+0000 mgr.y (mgr.34101) 1039 : cluster 0 pgmap v1640: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:48:36.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:36 smithi148 bash[79819]: cluster 2024-03-07T03:48:35.220617+0000 mgr.y (mgr.34101) 1039 : cluster 0 pgmap v1640: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:48:36.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:36 smithi195 bash[66542]: cluster 2024-03-07T03:48:35.220617+0000 mgr.y (mgr.34101) 1039 : cluster 0 pgmap v1640: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:48:37.549 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:48:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:48:37] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:48:38.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:38 smithi148 bash[79819]: cluster 2024-03-07T03:48:37.221453+0000 mgr.y (mgr.34101) 1040 : cluster 0 pgmap v1641: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:48:38.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:38 smithi148 bash[82222]: cluster 2024-03-07T03:48:37.221453+0000 mgr.y (mgr.34101) 1040 : cluster 0 pgmap v1641: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:48:38.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:38 smithi195 bash[66542]: cluster 2024-03-07T03:48:37.221453+0000 mgr.y (mgr.34101) 1040 : cluster 0 pgmap v1641: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-03-07T03:48:40.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:40 smithi148 bash[79819]: cluster 2024-03-07T03:48:39.222237+0000 mgr.y (mgr.34101) 1041 : cluster 0 pgmap v1642: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 199 KiB/s rd, 2.6 MiB/s wr, 30 op/s 2024-03-07T03:48:40.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:40 smithi148 bash[82222]: cluster 2024-03-07T03:48:39.222237+0000 mgr.y (mgr.34101) 1041 : cluster 0 pgmap v1642: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 199 KiB/s rd, 2.6 MiB/s wr, 30 op/s 2024-03-07T03:48:40.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:40 smithi195 bash[66542]: cluster 2024-03-07T03:48:39.222237+0000 mgr.y (mgr.34101) 1041 : cluster 0 pgmap v1642: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 199 KiB/s rd, 2.6 MiB/s wr, 30 op/s 2024-03-07T03:48:42.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:42 smithi195 bash[66542]: cluster 2024-03-07T03:48:41.222720+0000 mgr.y (mgr.34101) 1042 : cluster 0 pgmap v1643: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 1.7 MiB/s wr, 30 op/s 2024-03-07T03:48:42.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:42 smithi195 bash[66542]: audit 2024-03-07T03:48:42.350176+0000 mon.a (mon.0) 1805 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:42.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:48:42 smithi195 bash[59134]: ts=2024-03-07T03:48:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:42.648 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] EncryptedFlattenTest/7.ZeroOverlap (22317 ms) 2024-03-07T03:48:42.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:42 smithi148 bash[79819]: cluster 2024-03-07T03:48:41.222720+0000 mgr.y (mgr.34101) 1042 : cluster 0 pgmap v1643: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 1.7 MiB/s wr, 30 op/s 2024-03-07T03:48:42.799 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:42 smithi148 bash[79819]: audit 2024-03-07T03:48:42.350176+0000 mon.a (mon.0) 1805 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:42.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:42 smithi148 bash[82222]: cluster 2024-03-07T03:48:41.222720+0000 mgr.y (mgr.34101) 1042 : cluster 0 pgmap v1643: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 180 KiB/s rd, 1.7 MiB/s wr, 30 op/s 2024-03-07T03:48:42.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:42 smithi148 bash[82222]: audit 2024-03-07T03:48:42.350176+0000 mon.a (mon.0) 1805 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:43.555 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 7 tests from EncryptedFlattenTest/7 (155354 ms total) 2024-03-07T03:48:43.555 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:48:43.555 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 13 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-03-07T03:48:43.599 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-03-07T03:48:44.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:44 smithi195 bash[66542]: cluster 2024-03-07T03:48:43.223705+0000 mgr.y (mgr.34101) 1043 : cluster 0 pgmap v1644: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 1.3 MiB/s wr, 26 op/s 2024-03-07T03:48:44.821 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:44 smithi195 bash[66542]: cluster 2024-03-07T03:48:43.543366+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2024-03-07T03:48:45.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:44 smithi148 bash[79819]: cluster 2024-03-07T03:48:43.223705+0000 mgr.y (mgr.34101) 1043 : cluster 0 pgmap v1644: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 1.3 MiB/s wr, 26 op/s 2024-03-07T03:48:45.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:44 smithi148 bash[79819]: cluster 2024-03-07T03:48:43.543366+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2024-03-07T03:48:45.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:44 smithi148 bash[82222]: cluster 2024-03-07T03:48:43.223705+0000 mgr.y (mgr.34101) 1043 : cluster 0 pgmap v1644: 105 pgs: 105 active+clean; 472 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 1.3 MiB/s wr, 26 op/s 2024-03-07T03:48:45.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:44 smithi148 bash[82222]: cluster 2024-03-07T03:48:43.543366+0000 mon.a (mon.0) 1806 : cluster 0 osdmap e983: 8 total, 8 up, 8 in 2024-03-07T03:48:45.553 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[79819]: cluster 2024-03-07T03:48:44.549217+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[79819]: audit 2024-03-07T03:48:44.559370+0000 mon.a (mon.0) 1808 : audit 1 from='client.? 172.21.15.148:0/1873759304' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-44","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[79819]: audit 2024-03-07T03:48:45.543173+0000 mon.a (mon.0) 1809 : audit 1 from='client.? 172.21.15.148:0/1873759304' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[79819]: cluster 2024-03-07T03:48:45.551687+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[82222]: cluster 2024-03-07T03:48:44.549217+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[82222]: audit 2024-03-07T03:48:44.559370+0000 mon.a (mon.0) 1808 : audit 1 from='client.? 172.21.15.148:0/1873759304' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-44","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[82222]: audit 2024-03-07T03:48:45.543173+0000 mon.a (mon.0) 1809 : audit 1 from='client.? 172.21.15.148:0/1873759304' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:48:46.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:45 smithi148 bash[82222]: cluster 2024-03-07T03:48:45.551687+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2024-03-07T03:48:46.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:45 smithi195 bash[66542]: cluster 2024-03-07T03:48:44.549217+0000 mon.a (mon.0) 1807 : cluster 0 osdmap e984: 8 total, 8 up, 8 in 2024-03-07T03:48:46.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:45 smithi195 bash[66542]: audit 2024-03-07T03:48:44.559370+0000 mon.a (mon.0) 1808 : audit 1 from='client.? 172.21.15.148:0/1873759304' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-44","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:48:46.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:45 smithi195 bash[66542]: audit 2024-03-07T03:48:45.543173+0000 mon.a (mon.0) 1809 : audit 1 from='client.? 172.21.15.148:0/1873759304' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:48:46.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:45 smithi195 bash[66542]: cluster 2024-03-07T03:48:45.551687+0000 mon.a (mon.0) 1810 : cluster 0 osdmap e985: 8 total, 8 up, 8 in 2024-03-07T03:48:47.049 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:46 smithi148 bash[82222]: cluster 2024-03-07T03:48:45.224507+0000 mgr.y (mgr.34101) 1044 : cluster 0 pgmap v1647: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:48:47.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:46 smithi148 bash[82222]: cluster 2024-03-07T03:48:45.560519+0000 mon.a (mon.0) 1811 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:48:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:46 smithi148 bash[79819]: cluster 2024-03-07T03:48:45.224507+0000 mgr.y (mgr.34101) 1044 : cluster 0 pgmap v1647: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:48:47.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:46 smithi148 bash[79819]: cluster 2024-03-07T03:48:45.560519+0000 mon.a (mon.0) 1811 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:48:47.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:46 smithi195 bash[66542]: cluster 2024-03-07T03:48:45.224507+0000 mgr.y (mgr.34101) 1044 : cluster 0 pgmap v1647: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:48:47.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:46 smithi195 bash[66542]: cluster 2024-03-07T03:48:45.560519+0000 mon.a (mon.0) 1811 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:48:47.549 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:48:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:48:47] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-03-07T03:48:47.655 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [732082~10512,1909733~24112,3618521~101786,3887903~89553,5562060~77924,6290683~35602,7902192~90016,10036608~53297,10719940~75019,17663999~75723] 2024-03-07T03:48:48.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:47 smithi148 bash[79819]: cluster 2024-03-07T03:48:46.574072+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2024-03-07T03:48:48.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:47 smithi148 bash[82222]: cluster 2024-03-07T03:48:46.574072+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2024-03-07T03:48:48.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:47 smithi195 bash[66542]: cluster 2024-03-07T03:48:46.574072+0000 mon.a (mon.0) 1812 : cluster 0 osdmap e986: 8 total, 8 up, 8 in 2024-03-07T03:48:48.635 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [1786551~89575,3010502~78597,3283286~74782,5102119~41568,7150743~56289,14179026~86463,14692940~35590,14807284~39007,19876052~70039,20612160~9231] 2024-03-07T03:48:48.655 INFO:tasks.workunit.client.0.smithi148.stdout: diff was [1786551~89575,3010502~78597,3283286~74782,5102119~41568,7150743~56289,12582912~2263379,17739722~2881669] 2024-03-07T03:48:48.665 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5065 ms) 2024-03-07T03:48:48.665 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministic 2024-03-07T03:48:48.666 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:48:49.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:48 smithi148 bash[79819]: cluster 2024-03-07T03:48:47.225143+0000 mgr.y (mgr.34101) 1045 : cluster 0 pgmap v1650: 105 pgs: 7 creating+peering, 21 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:48:49.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:48 smithi148 bash[79819]: cluster 2024-03-07T03:48:47.578535+0000 mon.a (mon.0) 1813 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:48:49.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:48 smithi148 bash[79819]: cluster 2024-03-07T03:48:47.598895+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2024-03-07T03:48:49.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:48 smithi148 bash[82222]: cluster 2024-03-07T03:48:47.225143+0000 mgr.y (mgr.34101) 1045 : cluster 0 pgmap v1650: 105 pgs: 7 creating+peering, 21 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:48:49.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:48 smithi148 bash[82222]: cluster 2024-03-07T03:48:47.578535+0000 mon.a (mon.0) 1813 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:48:49.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:48 smithi148 bash[82222]: cluster 2024-03-07T03:48:47.598895+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2024-03-07T03:48:49.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:48 smithi195 bash[66542]: cluster 2024-03-07T03:48:47.225143+0000 mgr.y (mgr.34101) 1045 : cluster 0 pgmap v1650: 105 pgs: 7 creating+peering, 21 unknown, 77 active+clean; 582 KiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:48:49.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:48 smithi195 bash[66542]: cluster 2024-03-07T03:48:47.578535+0000 mon.a (mon.0) 1813 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:48:49.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:48 smithi195 bash[66542]: cluster 2024-03-07T03:48:47.598895+0000 mon.a (mon.0) 1814 : cluster 0 osdmap e987: 8 total, 8 up, 8 in 2024-03-07T03:48:50.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:49 smithi148 bash[79819]: cluster 2024-03-07T03:48:48.605932+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2024-03-07T03:48:50.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:49 smithi148 bash[82222]: cluster 2024-03-07T03:48:48.605932+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2024-03-07T03:48:50.070 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:49 smithi195 bash[66542]: cluster 2024-03-07T03:48:48.605932+0000 mon.a (mon.0) 1815 : cluster 0 osdmap e988: 8 total, 8 up, 8 in 2024-03-07T03:48:51.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:50 smithi148 bash[79819]: cluster 2024-03-07T03:48:49.226049+0000 mgr.y (mgr.34101) 1046 : cluster 0 pgmap v1653: 105 pgs: 7 creating+peering, 98 active+clean; 12 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 244 KiB/s wr, 33 op/s 2024-03-07T03:48:51.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:50 smithi148 bash[79819]: cluster 2024-03-07T03:48:49.617564+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2024-03-07T03:48:51.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:50 smithi148 bash[82222]: cluster 2024-03-07T03:48:49.226049+0000 mgr.y (mgr.34101) 1046 : cluster 0 pgmap v1653: 105 pgs: 7 creating+peering, 98 active+clean; 12 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 244 KiB/s wr, 33 op/s 2024-03-07T03:48:51.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:50 smithi148 bash[82222]: cluster 2024-03-07T03:48:49.617564+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2024-03-07T03:48:51.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:50 smithi195 bash[66542]: cluster 2024-03-07T03:48:49.226049+0000 mgr.y (mgr.34101) 1046 : cluster 0 pgmap v1653: 105 pgs: 7 creating+peering, 98 active+clean; 12 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 18 KiB/s rd, 244 KiB/s wr, 33 op/s 2024-03-07T03:48:51.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:50 smithi195 bash[66542]: cluster 2024-03-07T03:48:49.617564+0000 mon.a (mon.0) 1816 : cluster 0 osdmap e989: 8 total, 8 up, 8 in 2024-03-07T03:48:51.688 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministic (3023 ms) 2024-03-07T03:48:51.688 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicPP 2024-03-07T03:48:51.688 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:48:52.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:51 smithi148 bash[79819]: cluster 2024-03-07T03:48:50.618447+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2024-03-07T03:48:52.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:51 smithi148 bash[82222]: cluster 2024-03-07T03:48:50.618447+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2024-03-07T03:48:52.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:51 smithi195 bash[66542]: cluster 2024-03-07T03:48:50.618447+0000 mon.a (mon.0) 1817 : cluster 0 osdmap e990: 8 total, 8 up, 8 in 2024-03-07T03:48:52.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:48:52 smithi195 bash[59134]: ts=2024-03-07T03:48:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:48:53.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:52 smithi148 bash[79819]: cluster 2024-03-07T03:48:51.226759+0000 mgr.y (mgr.34101) 1047 : cluster 0 pgmap v1656: 105 pgs: 105 active+clean; 12 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 245 KiB/s wr, 48 op/s 2024-03-07T03:48:53.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:52 smithi148 bash[79819]: cluster 2024-03-07T03:48:51.635796+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2024-03-07T03:48:53.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:52 smithi148 bash[82222]: cluster 2024-03-07T03:48:51.226759+0000 mgr.y (mgr.34101) 1047 : cluster 0 pgmap v1656: 105 pgs: 105 active+clean; 12 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 245 KiB/s wr, 48 op/s 2024-03-07T03:48:53.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:52 smithi148 bash[82222]: cluster 2024-03-07T03:48:51.635796+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2024-03-07T03:48:53.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:52 smithi195 bash[66542]: cluster 2024-03-07T03:48:51.226759+0000 mgr.y (mgr.34101) 1047 : cluster 0 pgmap v1656: 105 pgs: 105 active+clean; 12 MiB data, 597 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 245 KiB/s wr, 48 op/s 2024-03-07T03:48:53.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:52 smithi195 bash[66542]: cluster 2024-03-07T03:48:51.635796+0000 mon.a (mon.0) 1818 : cluster 0 osdmap e991: 8 total, 8 up, 8 in 2024-03-07T03:48:54.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:53 smithi148 bash[79819]: cluster 2024-03-07T03:48:52.650247+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2024-03-07T03:48:54.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:53 smithi148 bash[82222]: cluster 2024-03-07T03:48:52.650247+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2024-03-07T03:48:54.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:53 smithi195 bash[66542]: cluster 2024-03-07T03:48:52.650247+0000 mon.a (mon.0) 1819 : cluster 0 osdmap e992: 8 total, 8 up, 8 in 2024-03-07T03:48:54.712 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicPP (3024 ms) 2024-03-07T03:48:54.712 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-03-07T03:48:54.712 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:48:55.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:54 smithi148 bash[79819]: cluster 2024-03-07T03:48:53.227733+0000 mgr.y (mgr.34101) 1048 : cluster 0 pgmap v1659: 105 pgs: 105 active+clean; 16 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 65 KiB/s wr, 51 op/s 2024-03-07T03:48:55.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:54 smithi148 bash[79819]: cluster 2024-03-07T03:48:53.659988+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2024-03-07T03:48:55.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:54 smithi148 bash[82222]: cluster 2024-03-07T03:48:53.227733+0000 mgr.y (mgr.34101) 1048 : cluster 0 pgmap v1659: 105 pgs: 105 active+clean; 16 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 65 KiB/s wr, 51 op/s 2024-03-07T03:48:55.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:54 smithi148 bash[82222]: cluster 2024-03-07T03:48:53.659988+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2024-03-07T03:48:55.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:54 smithi195 bash[66542]: cluster 2024-03-07T03:48:53.227733+0000 mgr.y (mgr.34101) 1048 : cluster 0 pgmap v1659: 105 pgs: 105 active+clean; 16 MiB data, 599 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 65 KiB/s wr, 51 op/s 2024-03-07T03:48:55.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:54 smithi195 bash[66542]: cluster 2024-03-07T03:48:53.659988+0000 mon.a (mon.0) 1820 : cluster 0 osdmap e993: 8 total, 8 up, 8 in 2024-03-07T03:48:56.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:55 smithi148 bash[79819]: cluster 2024-03-07T03:48:54.661448+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2024-03-07T03:48:56.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:55 smithi148 bash[82222]: cluster 2024-03-07T03:48:54.661448+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2024-03-07T03:48:56.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:55 smithi195 bash[66542]: cluster 2024-03-07T03:48:54.661448+0000 mon.a (mon.0) 1821 : cluster 0 osdmap e994: 8 total, 8 up, 8 in 2024-03-07T03:48:57.049 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:56 smithi148 bash[79819]: cluster 2024-03-07T03:48:55.228520+0000 mgr.y (mgr.34101) 1049 : cluster 0 pgmap v1662: 105 pgs: 105 active+clean; 16 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 66 KiB/s wr, 50 op/s 2024-03-07T03:48:57.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:56 smithi148 bash[79819]: cluster 2024-03-07T03:48:55.682359+0000 mon.a (mon.0) 1822 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2024-03-07T03:48:57.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:56 smithi148 bash[82222]: cluster 2024-03-07T03:48:55.228520+0000 mgr.y (mgr.34101) 1049 : cluster 0 pgmap v1662: 105 pgs: 105 active+clean; 16 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 66 KiB/s wr, 50 op/s 2024-03-07T03:48:57.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:56 smithi148 bash[82222]: cluster 2024-03-07T03:48:55.682359+0000 mon.a (mon.0) 1822 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2024-03-07T03:48:57.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:56 smithi195 bash[66542]: cluster 2024-03-07T03:48:55.228520+0000 mgr.y (mgr.34101) 1049 : cluster 0 pgmap v1662: 105 pgs: 105 active+clean; 16 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 66 KiB/s wr, 50 op/s 2024-03-07T03:48:57.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:56 smithi195 bash[66542]: cluster 2024-03-07T03:48:55.682359+0000 mon.a (mon.0) 1822 : cluster 0 osdmap e995: 8 total, 8 up, 8 in 2024-03-07T03:48:57.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:48:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:48:57] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.43.0" 2024-03-07T03:48:57.773 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3061 ms) 2024-03-07T03:48:57.773 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-03-07T03:48:57.773 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:48:57.910 INFO:tasks.workunit.client.0.smithi148.stdout: i=0 exists [57733079~254643,96448457~6460333,106634891~27679,115822987~5082362,132501903~3808928,139808196~7836872,251563290~2822495,312073576~8674750,336994803~2847404] wrote [57733079~254643,96448457~6460333,106634891~27679,115822987~5082362,132501903~3808928,139808196~7836872,251563290~2822495,312073576~8674750,336994803~2847404] 2024-03-07T03:48:58.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:57 smithi148 bash[79819]: cluster 2024-03-07T03:48:56.688443+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2024-03-07T03:48:58.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:57 smithi148 bash[79819]: audit 2024-03-07T03:48:57.350772+0000 mon.a (mon.0) 1824 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:58.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:57 smithi148 bash[82222]: cluster 2024-03-07T03:48:56.688443+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2024-03-07T03:48:58.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:57 smithi148 bash[82222]: audit 2024-03-07T03:48:57.350772+0000 mon.a (mon.0) 1824 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:58.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:57 smithi195 bash[66542]: cluster 2024-03-07T03:48:56.688443+0000 mon.a (mon.0) 1823 : cluster 0 osdmap e996: 8 total, 8 up, 8 in 2024-03-07T03:48:58.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:57 smithi195 bash[66542]: audit 2024-03-07T03:48:57.350772+0000 mon.a (mon.0) 1824 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:48:58.881 INFO:tasks.workunit.client.0.smithi148.stdout: i=1 exists [26837245~5573006,50347152~272229,57733079~254643,76104975~5291461,96448457~6460333,106634891~27679,115822987~5082362,132501903~3808928,139808196~8270093,176574882~434201,230220240~7486763,241323653~2754980,251563290~2822495,264509899~7106289,306536384~4114836,312073576~8674750,336994803~2847404,356816538~6725480] wrote [26837245~5573006,50347152~272229,76104975~5291461,146288090~1790199,176574882~434201,230220240~7486763,241323653~2754980,264509899~7106289,306536384~4114836,356816538~6725480] 2024-03-07T03:48:59.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:58 smithi148 bash[79819]: cluster 2024-03-07T03:48:57.229171+0000 mgr.y (mgr.34101) 1050 : cluster 0 pgmap v1665: 105 pgs: 105 active+clean; 16 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 6.2 KiB/s wr, 50 op/s 2024-03-07T03:48:59.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:58 smithi148 bash[79819]: cluster 2024-03-07T03:48:57.712077+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2024-03-07T03:48:59.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:58 smithi148 bash[79819]: cluster 2024-03-07T03:48:57.770389+0000 mon.a (mon.0) 1826 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2024-03-07T03:48:59.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:58 smithi148 bash[82222]: cluster 2024-03-07T03:48:57.229171+0000 mgr.y (mgr.34101) 1050 : cluster 0 pgmap v1665: 105 pgs: 105 active+clean; 16 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 6.2 KiB/s wr, 50 op/s 2024-03-07T03:48:59.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:58 smithi148 bash[82222]: cluster 2024-03-07T03:48:57.712077+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2024-03-07T03:48:59.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:58 smithi148 bash[82222]: cluster 2024-03-07T03:48:57.770389+0000 mon.a (mon.0) 1826 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2024-03-07T03:48:59.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:58 smithi195 bash[66542]: cluster 2024-03-07T03:48:57.229171+0000 mgr.y (mgr.34101) 1050 : cluster 0 pgmap v1665: 105 pgs: 105 active+clean; 16 MiB data, 600 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 6.2 KiB/s wr, 50 op/s 2024-03-07T03:48:59.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:58 smithi195 bash[66542]: cluster 2024-03-07T03:48:57.712077+0000 mon.a (mon.0) 1825 : cluster 0 osdmap e997: 8 total, 8 up, 8 in 2024-03-07T03:48:59.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:58 smithi195 bash[66542]: cluster 2024-03-07T03:48:57.770389+0000 mon.a (mon.0) 1826 : cluster 0 osdmap e998: 8 total, 8 up, 8 in 2024-03-07T03:48:59.977 INFO:tasks.workunit.client.0.smithi148.stdout: i=2 exists [6453699~2873285,17913684~4306597,22356670~10053581,50347152~272229,57733079~254643,76104975~5291461,96448457~6460333,106634891~27679,115822987~5082362,132501903~15576386,176574882~434201,230220240~7486763,241323653~2754980,251563290~2822495,264509899~7106289,306536384~4114836,312073576~8674750,322318926~5618689,336994803~2847404,356612098~7305062,404168647~1959084] wrote [6453699~2873285,17913684~4306597,22356670~5955932,135946328~6583779,312442734~5263519,322318926~5618689,356612098~7305062,404168647~1959084] 2024-03-07T03:49:00.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:59 smithi148 bash[79819]: cluster 2024-03-07T03:48:58.772786+0000 mon.a (mon.0) 1827 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2024-03-07T03:49:00.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:48:59 smithi148 bash[79819]: audit 2024-03-07T03:48:58.860642+0000 mon.a (mon.0) 1828 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:00.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:59 smithi148 bash[82222]: cluster 2024-03-07T03:48:58.772786+0000 mon.a (mon.0) 1827 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2024-03-07T03:49:00.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:48:59 smithi148 bash[82222]: audit 2024-03-07T03:48:58.860642+0000 mon.a (mon.0) 1828 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:00.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:59 smithi195 bash[66542]: cluster 2024-03-07T03:48:58.772786+0000 mon.a (mon.0) 1827 : cluster 0 osdmap e999: 8 total, 8 up, 8 in 2024-03-07T03:49:00.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:48:59 smithi195 bash[66542]: audit 2024-03-07T03:48:58.860642+0000 mon.a (mon.0) 1828 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:00.983 INFO:tasks.workunit.client.0.smithi148.stdout: i=3 exists [3227891~6118297,17913684~4306597,22356670~10053581,50347152~272229,57733079~254643,76104975~5291461,87450726~3021868,96448457~6460333,106634891~27679,111365223~3136346,115063698~5841651,124635782~1926009,132501903~15576386,176574882~434201,182263066~5648763,230220240~7486763,241323653~2754980,251563290~2822495,264509899~7106289,306536384~4114836,312073576~8674750,322318926~5618689,336994803~2847404,346627350~6845553,356612098~7305062,367989998~11463074,404168647~1959084] wrote [3227891~6118297,87450726~3021868,111365223~3136346,115063698~1259632,124635782~1926009,182263066~5648763,346627350~6845553,367989998~11463074] 2024-03-07T03:49:01.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:00 smithi148 bash[79819]: cluster 2024-03-07T03:48:59.230001+0000 mgr.y (mgr.34101) 1051 : cluster 0 pgmap v1669: 105 pgs: 105 active+clean; 64 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 8.6 MiB/s wr, 138 op/s 2024-03-07T03:49:01.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:00 smithi148 bash[79819]: cluster 2024-03-07T03:48:59.876022+0000 mon.a (mon.0) 1829 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2024-03-07T03:49:01.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:00 smithi148 bash[82222]: cluster 2024-03-07T03:48:59.230001+0000 mgr.y (mgr.34101) 1051 : cluster 0 pgmap v1669: 105 pgs: 105 active+clean; 64 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 8.6 MiB/s wr, 138 op/s 2024-03-07T03:49:01.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:00 smithi148 bash[82222]: cluster 2024-03-07T03:48:59.876022+0000 mon.a (mon.0) 1829 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2024-03-07T03:49:01.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:00 smithi195 bash[66542]: cluster 2024-03-07T03:48:59.230001+0000 mgr.y (mgr.34101) 1051 : cluster 0 pgmap v1669: 105 pgs: 105 active+clean; 64 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 85 KiB/s rd, 8.6 MiB/s wr, 138 op/s 2024-03-07T03:49:01.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:00 smithi195 bash[66542]: cluster 2024-03-07T03:48:59.876022+0000 mon.a (mon.0) 1829 : cluster 0 osdmap e1000: 8 total, 8 up, 8 in 2024-03-07T03:49:01.998 INFO:tasks.workunit.client.0.smithi148.stdout: i=4 exists [3227891~6118297,17913684~4306597,22356670~10053581,47457196~10921189,70050257~2762559,75213635~6182801,86189502~5165955,96448457~6460333,106634891~27679,111365223~3136346,115063698~5841651,124635782~1926009,132501903~22079942,176574882~434201,178456707~9455122,230220240~8131429,241323653~2754980,251563290~2822495,264509899~9916446,306536384~4114836,312073576~8674750,322318926~5618689,336994803~2847404,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] wrote [47457196~10921189,70050257~2762559,75213635~1919358,86189502~5165955,146733854~7847991,178456707~4628361,233433683~4917986,266839104~7587241,380459900~1937696] 2024-03-07T03:49:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:01 smithi148 bash[79819]: cluster 2024-03-07T03:49:00.886513+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2024-03-07T03:49:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:01 smithi148 bash[79819]: cluster 2024-03-07T03:49:01.230656+0000 mgr.y (mgr.34101) 1052 : cluster 0 pgmap v1672: 105 pgs: 105 active+clean; 85 MiB data, 705 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 14 MiB/s wr, 257 op/s 2024-03-07T03:49:02.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:01 smithi148 bash[82222]: cluster 2024-03-07T03:49:00.886513+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2024-03-07T03:49:02.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:01 smithi148 bash[82222]: cluster 2024-03-07T03:49:01.230656+0000 mgr.y (mgr.34101) 1052 : cluster 0 pgmap v1672: 105 pgs: 105 active+clean; 85 MiB data, 705 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 14 MiB/s wr, 257 op/s 2024-03-07T03:49:02.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:01 smithi195 bash[66542]: cluster 2024-03-07T03:49:00.886513+0000 mon.a (mon.0) 1830 : cluster 0 osdmap e1001: 8 total, 8 up, 8 in 2024-03-07T03:49:02.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:01 smithi195 bash[66542]: cluster 2024-03-07T03:49:01.230656+0000 mgr.y (mgr.34101) 1052 : cluster 0 pgmap v1672: 105 pgs: 105 active+clean; 85 MiB data, 705 MiB used, 715 GiB / 715 GiB avail; 157 KiB/s rd, 14 MiB/s wr, 257 op/s 2024-03-07T03:49:02.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:49:02 smithi195 bash[59134]: ts=2024-03-07T03:49:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:03.028 INFO:tasks.workunit.client.0.smithi148.stdout: i=5 exists [1668238~7677950,17913684~4306597,22356670~10053581,47457196~10921189,70050257~11346179,86189502~5165955,96448457~6460333,106634891~27679,111365223~15196568,131400499~23181346,176574882~434201,178456707~9455122,230220240~8131429,241323653~2754980,251563290~2822495,264509899~9916446,293107900~4881608,306536384~4114836,312073576~8674750,322318926~5618689,331579789~8262418,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] wrote [1668238~4944408,72347956~6121902,101146541~1069076,114140049~10701635,131400499~8153873,293107900~4881608,331579789~6133207,349128013~3657743] 2024-03-07T03:49:03.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:02 smithi148 bash[79819]: cluster 2024-03-07T03:49:01.897699+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2024-03-07T03:49:03.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:02 smithi148 bash[82222]: cluster 2024-03-07T03:49:01.897699+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2024-03-07T03:49:03.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:02 smithi195 bash[66542]: cluster 2024-03-07T03:49:01.897699+0000 mon.a (mon.0) 1831 : cluster 0 osdmap e1002: 8 total, 8 up, 8 in 2024-03-07T03:49:04.060 INFO:tasks.workunit.client.0.smithi148.stdout: i=6 exists [1668238~7677950,16319008~5901273,22356670~10053581,38735630~5634426,47457196~10921189,70050257~11346179,86189502~5165955,96448457~6460333,106634891~27679,111365223~15196568,131400499~23181346,176574882~434201,178456707~9455122,217208288~1229660,230220240~8131429,241323653~2754980,251563290~2822495,260950876~13475469,293107900~4881608,306536384~14211942,322318926~5618689,331579789~8262418,345377430~8095473,356612098~7305062,367989998~11463074,380459900~1937696,388314730~1562479,404168647~1959084] wrote [3758582~4326861,16319008~5418293,38735630~5634426,217208288~1229660,260950876~7549629,308958519~7144889,333531803~4962728,345377430~7491289,388314730~1562479] 2024-03-07T03:49:04.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:03 smithi148 bash[79819]: cluster 2024-03-07T03:49:02.899040+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2024-03-07T03:49:04.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:03 smithi148 bash[79819]: cluster 2024-03-07T03:49:03.231551+0000 mgr.y (mgr.34101) 1053 : cluster 0 pgmap v1675: 105 pgs: 105 active+clean; 236 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 38 MiB/s wr, 190 op/s 2024-03-07T03:49:04.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:03 smithi148 bash[82222]: cluster 2024-03-07T03:49:02.899040+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2024-03-07T03:49:04.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:03 smithi148 bash[82222]: cluster 2024-03-07T03:49:03.231551+0000 mgr.y (mgr.34101) 1053 : cluster 0 pgmap v1675: 105 pgs: 105 active+clean; 236 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 38 MiB/s wr, 190 op/s 2024-03-07T03:49:04.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:03 smithi195 bash[66542]: cluster 2024-03-07T03:49:02.899040+0000 mon.a (mon.0) 1832 : cluster 0 osdmap e1003: 8 total, 8 up, 8 in 2024-03-07T03:49:04.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:03 smithi195 bash[66542]: cluster 2024-03-07T03:49:03.231551+0000 mgr.y (mgr.34101) 1053 : cluster 0 pgmap v1675: 105 pgs: 105 active+clean; 236 MiB data, 1016 MiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 38 MiB/s wr, 190 op/s 2024-03-07T03:49:05.018 INFO:tasks.workunit.client.0.smithi148.stdout: i=7 exists [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,70050257~11346179,86189502~5165955,96448457~6460333,106634891~27679,111365223~15196568,131400499~23181346,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~4186570,230220240~8131429,241323653~2754980,251563290~2822495,260950876~13475469,286966749~11022759,306536384~14211942,322318926~5618689,331579789~8793095,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] wrote [20871198~869054,43299722~3090762,54508011~6897241,136109713~7293054,172622639~1160618,219907509~4186570,286966749~7178032,338641176~1731708,365771831~4427582,394784133~2473510] 2024-03-07T03:49:05.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:04 smithi148 bash[79819]: cluster 2024-03-07T03:49:03.921425+0000 mon.a (mon.0) 1833 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2024-03-07T03:49:05.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:04 smithi148 bash[82222]: cluster 2024-03-07T03:49:03.921425+0000 mon.a (mon.0) 1833 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2024-03-07T03:49:05.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:04 smithi195 bash[66542]: cluster 2024-03-07T03:49:03.921425+0000 mon.a (mon.0) 1833 : cluster 0 osdmap e1004: 8 total, 8 up, 8 in 2024-03-07T03:49:06.085 INFO:tasks.workunit.client.0.smithi148.stdout: i=8 exists [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,70050257~11346179,86189502~5165955,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~23181346,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~4186570,230220240~13858393,251563290~2822495,260950876~13475469,283198497~14791011,306536384~14211942,322318926~5618689,331500133~8872751,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] wrote [57610260~405200,71880073~7248390,107640648~1371632,135426417~7906202,235744159~5943951,283198497~9374773,311846963~6942376,331500133~3539963,369938543~5581530] 2024-03-07T03:49:06.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:05 smithi148 bash[79819]: cluster 2024-03-07T03:49:04.930190+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2024-03-07T03:49:06.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:05 smithi148 bash[79819]: audit 2024-03-07T03:49:05.077603+0000 mon.a (mon.0) 1835 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:49:06.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:05 smithi148 bash[79819]: cluster 2024-03-07T03:49:05.232094+0000 mgr.y (mgr.34101) 1054 : cluster 0 pgmap v1678: 105 pgs: 105 active+clean; 310 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 48 MiB/s wr, 284 op/s 2024-03-07T03:49:06.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:05 smithi148 bash[82222]: cluster 2024-03-07T03:49:04.930190+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2024-03-07T03:49:06.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:05 smithi148 bash[82222]: audit 2024-03-07T03:49:05.077603+0000 mon.a (mon.0) 1835 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:49:06.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:05 smithi148 bash[82222]: cluster 2024-03-07T03:49:05.232094+0000 mgr.y (mgr.34101) 1054 : cluster 0 pgmap v1678: 105 pgs: 105 active+clean; 310 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 48 MiB/s wr, 284 op/s 2024-03-07T03:49:06.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:05 smithi195 bash[66542]: cluster 2024-03-07T03:49:04.930190+0000 mon.a (mon.0) 1834 : cluster 0 osdmap e1005: 8 total, 8 up, 8 in 2024-03-07T03:49:06.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:05 smithi195 bash[66542]: audit 2024-03-07T03:49:05.077603+0000 mon.a (mon.0) 1835 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:49:06.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:05 smithi195 bash[66542]: cluster 2024-03-07T03:49:05.232094+0000 mgr.y (mgr.34101) 1054 : cluster 0 pgmap v1678: 105 pgs: 105 active+clean; 310 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 171 KiB/s rd, 48 MiB/s wr, 284 op/s 2024-03-07T03:49:07.060 INFO:tasks.workunit.client.0.smithi148.stdout: i=9 exists [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,68844351~12552085,86189502~5165955,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~23181346,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~8387335,230220240~13858393,251563290~2822495,258627946~17857475,278150166~19839342,306536384~14211942,322318926~5618689,331500133~8872751,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] wrote [68844351~5011596,143458483~109788,220902876~7391968,258627946~3865095,268107668~635365,271980343~4505078,278150166~7655540,357034236~4493146] 2024-03-07T03:49:07.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:06 smithi148 bash[79819]: cluster 2024-03-07T03:49:05.944045+0000 mon.a (mon.0) 1836 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2024-03-07T03:49:07.300 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:49:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:49:07] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.43.0" 2024-03-07T03:49:07.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:06 smithi148 bash[82222]: cluster 2024-03-07T03:49:05.944045+0000 mon.a (mon.0) 1836 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2024-03-07T03:49:07.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:06 smithi195 bash[66542]: cluster 2024-03-07T03:49:05.944045+0000 mon.a (mon.0) 1836 : cluster 0 osdmap e1006: 8 total, 8 up, 8 in 2024-03-07T03:49:08.055 INFO:tasks.workunit.client.0.smithi148.stdout: i=10 exists [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~23181346,172622639~1160618,175143458~13391092,217208288~1229660,219907509~8387335,230220240~13858393,248347320~6038465,258627946~18162098,278150166~19839342,306536384~14211942,322318926~5618689,331500133~8872751,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] wrote [27243122~8307066,42537032~1697190,78559315~6456423,112841617~4261226,175143458~3505007,187362711~1171839,248347320~4168563,274300269~2489775] 2024-03-07T03:49:08.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:07 smithi148 bash[79819]: cluster 2024-03-07T03:49:06.949704+0000 mon.a (mon.0) 1837 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2024-03-07T03:49:08.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:07 smithi148 bash[79819]: cluster 2024-03-07T03:49:07.232708+0000 mgr.y (mgr.34101) 1055 : cluster 0 pgmap v1681: 105 pgs: 105 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 126 KiB/s rd, 28 MiB/s wr, 216 op/s 2024-03-07T03:49:08.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:07 smithi148 bash[82222]: cluster 2024-03-07T03:49:06.949704+0000 mon.a (mon.0) 1837 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2024-03-07T03:49:08.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:07 smithi148 bash[82222]: cluster 2024-03-07T03:49:07.232708+0000 mgr.y (mgr.34101) 1055 : cluster 0 pgmap v1681: 105 pgs: 105 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 126 KiB/s rd, 28 MiB/s wr, 216 op/s 2024-03-07T03:49:08.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:07 smithi195 bash[66542]: cluster 2024-03-07T03:49:06.949704+0000 mon.a (mon.0) 1837 : cluster 0 osdmap e1007: 8 total, 8 up, 8 in 2024-03-07T03:49:08.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:07 smithi195 bash[66542]: cluster 2024-03-07T03:49:07.232708+0000 mgr.y (mgr.34101) 1055 : cluster 0 pgmap v1681: 105 pgs: 105 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 126 KiB/s rd, 28 MiB/s wr, 216 op/s 2024-03-07T03:49:09.055 INFO:tasks.workunit.client.0.smithi148.stdout: i=11 exists [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,172622639~1160618,175143458~13391092,217208288~1229660,219907509~24171124,248347320~6038465,258627946~18162098,278150166~19839342,306536384~14211942,322318926~5618689,331500133~8872751,345377430~8095473,356612098~22840974,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] wrote [56594107~6132924,66725534~1527286,154546637~1285898,224740997~7252231,249454878~3683227,292812639~4148335,322402909~5512743,362085616~6636924,375723852~2962468] 2024-03-07T03:49:09.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:08 smithi148 bash[79819]: cluster 2024-03-07T03:49:07.958694+0000 mon.a (mon.0) 1838 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2024-03-07T03:49:09.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:08 smithi148 bash[82222]: cluster 2024-03-07T03:49:07.958694+0000 mon.a (mon.0) 1838 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2024-03-07T03:49:09.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:08 smithi195 bash[66542]: cluster 2024-03-07T03:49:07.958694+0000 mon.a (mon.0) 1838 : cluster 0 osdmap e1008: 8 total, 8 up, 8 in 2024-03-07T03:49:10.088 INFO:tasks.workunit.client.0.smithi148.stdout: i=12 exists [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,172622639~1160618,175143458~13391092,217208288~1229660,219907509~28395341,248347320~6038465,258627946~18162098,278150166~19839342,300302651~2270395,306536384~14211942,322318926~5618689,331500133~8872751,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394784133~6342267,404168647~1959084] wrote [20376114~4304906,31904019~2211031,75283715~3751471,226244801~6641063,243876757~4426093,282524789~1540418,287521464~6807957,300302651~2270395,386954373~6992617,396810984~4315416] 2024-03-07T03:49:10.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:09 smithi148 bash[79819]: cluster 2024-03-07T03:49:08.960802+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2024-03-07T03:49:10.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:09 smithi148 bash[79819]: cluster 2024-03-07T03:49:09.233575+0000 mgr.y (mgr.34101) 1056 : cluster 0 pgmap v1684: 105 pgs: 105 active+clean; 518 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 50 MiB/s wr, 126 op/s 2024-03-07T03:49:10.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:09 smithi148 bash[82222]: cluster 2024-03-07T03:49:08.960802+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2024-03-07T03:49:10.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:09 smithi148 bash[82222]: cluster 2024-03-07T03:49:09.233575+0000 mgr.y (mgr.34101) 1056 : cluster 0 pgmap v1684: 105 pgs: 105 active+clean; 518 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 50 MiB/s wr, 126 op/s 2024-03-07T03:49:10.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:09 smithi195 bash[66542]: cluster 2024-03-07T03:49:08.960802+0000 mon.a (mon.0) 1839 : cluster 0 osdmap e1009: 8 total, 8 up, 8 in 2024-03-07T03:49:10.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:09 smithi195 bash[66542]: cluster 2024-03-07T03:49:09.233575+0000 mgr.y (mgr.34101) 1056 : cluster 0 pgmap v1684: 105 pgs: 105 active+clean; 518 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 76 KiB/s rd, 50 MiB/s wr, 126 op/s 2024-03-07T03:49:11.053 INFO:tasks.workunit.client.0.smithi148.stdout: i=13 exists [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,158985649~3711001,172622639~1160618,175143458~13391092,217208288~1229660,219907509~28395341,248347320~6038465,258627946~18162098,278150166~19839342,300302651~2270395,306536384~14211942,322318926~5618689,330544483~9828401,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] wrote [18888173~3944668,73270615~516924,147091666~5234760,158985649~3711001,180903040~3278595,324374225~70598,330544483~7413135,366918734~3516825,388807829~4661986,394504318~583478] 2024-03-07T03:49:11.299 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:10 smithi148 bash[79819]: cluster 2024-03-07T03:49:09.986753+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2024-03-07T03:49:11.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:10 smithi148 bash[79819]: audit 2024-03-07T03:49:10.815275+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:11.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:10 smithi148 bash[79819]: audit 2024-03-07T03:49:10.825334+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:11.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:10 smithi148 bash[82222]: cluster 2024-03-07T03:49:09.986753+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2024-03-07T03:49:11.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:10 smithi148 bash[82222]: audit 2024-03-07T03:49:10.815275+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:11.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:10 smithi148 bash[82222]: audit 2024-03-07T03:49:10.825334+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:11.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:10 smithi195 bash[66542]: cluster 2024-03-07T03:49:09.986753+0000 mon.a (mon.0) 1840 : cluster 0 osdmap e1010: 8 total, 8 up, 8 in 2024-03-07T03:49:11.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:10 smithi195 bash[66542]: audit 2024-03-07T03:49:10.815275+0000 mon.a (mon.0) 1841 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:11.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:10 smithi195 bash[66542]: audit 2024-03-07T03:49:10.825334+0000 mon.a (mon.0) 1842 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.094 INFO:tasks.workunit.client.0.smithi148.stdout: i=14 exists [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,158985649~3711001,172622639~1160618,175143458~13391092,207908122~5517064,217208288~1229660,219907509~28395341,248347320~6038465,258627946~18162098,278150166~19839342,300302651~2270395,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] wrote [69963884~5451504,93923603~1285999,155324570~457207,207908122~5517064,263519614~632790,290524600~5496444,315737520~6846870,324289746~6442358,358137084~5551677,396876058~2804493] 2024-03-07T03:49:12.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[79819]: cluster 2024-03-07T03:49:10.984222+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2024-03-07T03:49:12.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[79819]: audit 2024-03-07T03:49:11.171256+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[79819]: audit 2024-03-07T03:49:11.182940+0000 mon.a (mon.0) 1845 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[79819]: cluster 2024-03-07T03:49:11.234056+0000 mgr.y (mgr.34101) 1057 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 548 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 45 MiB/s wr, 311 op/s 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[79819]: audit 2024-03-07T03:49:11.818625+0000 mon.a (mon.0) 1846 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[79819]: audit 2024-03-07T03:49:11.820824+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[79819]: audit 2024-03-07T03:49:11.831821+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[82222]: cluster 2024-03-07T03:49:10.984222+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[82222]: audit 2024-03-07T03:49:11.171256+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[82222]: audit 2024-03-07T03:49:11.182940+0000 mon.a (mon.0) 1845 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[82222]: cluster 2024-03-07T03:49:11.234056+0000 mgr.y (mgr.34101) 1057 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 548 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 45 MiB/s wr, 311 op/s 2024-03-07T03:49:12.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[82222]: audit 2024-03-07T03:49:11.818625+0000 mon.a (mon.0) 1846 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:49:12.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[82222]: audit 2024-03-07T03:49:11.820824+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:49:12.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:12 smithi148 bash[82222]: audit 2024-03-07T03:49:11.831821+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[66542]: cluster 2024-03-07T03:49:10.984222+0000 mon.a (mon.0) 1843 : cluster 0 osdmap e1011: 8 total, 8 up, 8 in 2024-03-07T03:49:12.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[66542]: audit 2024-03-07T03:49:11.171256+0000 mon.a (mon.0) 1844 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[66542]: audit 2024-03-07T03:49:11.182940+0000 mon.a (mon.0) 1845 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[66542]: cluster 2024-03-07T03:49:11.234056+0000 mgr.y (mgr.34101) 1057 : cluster 0 pgmap v1687: 105 pgs: 105 active+clean; 548 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 45 MiB/s wr, 311 op/s 2024-03-07T03:49:12.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[66542]: audit 2024-03-07T03:49:11.818625+0000 mon.a (mon.0) 1846 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:49:12.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[66542]: audit 2024-03-07T03:49:11.820824+0000 mon.a (mon.0) 1847 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:49:12.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[66542]: audit 2024-03-07T03:49:11.831821+0000 mon.a (mon.0) 1848 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:49:12.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:49:12 smithi195 bash[59134]: ts=2024-03-07T03:49:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:13.106 INFO:tasks.workunit.client.0.smithi148.stdout: i=15 exists [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,158985649~3711001,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~6038465,258627946~18162098,278150166~19839342,300302651~2270395,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] wrote [15396897~3963058,35879825~1733789,181822460~3589030,197418188~7031285,221858299~3493903,265268048~2824484,323482505~2351426,334948415~2373553,337602204~2635002,367576613~5297086] 2024-03-07T03:49:13.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:13 smithi148 bash[79819]: cluster 2024-03-07T03:49:11.995757+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2024-03-07T03:49:13.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:13 smithi148 bash[79819]: audit 2024-03-07T03:49:12.351286+0000 mon.a (mon.0) 1850 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:13.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:13 smithi148 bash[82222]: cluster 2024-03-07T03:49:11.995757+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2024-03-07T03:49:13.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:13 smithi148 bash[82222]: audit 2024-03-07T03:49:12.351286+0000 mon.a (mon.0) 1850 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:13.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:13 smithi195 bash[66542]: cluster 2024-03-07T03:49:11.995757+0000 mon.a (mon.0) 1849 : cluster 0 osdmap e1012: 8 total, 8 up, 8 in 2024-03-07T03:49:13.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:13 smithi195 bash[66542]: audit 2024-03-07T03:49:12.351286+0000 mon.a (mon.0) 1850 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:14.132 INFO:tasks.workunit.client.0.smithi148.stdout: i=16 exists [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~6038465,258627946~18162098,278150166~19839342,300302651~2270395,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] wrote [3099290~416434,78357669~8827934,118268023~5048976,162665439~6006638,223024139~2214744,233865617~936151,263372186~5743786,312404032~1018043,371688229~1167625] 2024-03-07T03:49:14.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:14 smithi148 bash[79819]: cluster 2024-03-07T03:49:13.014390+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2024-03-07T03:49:14.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:14 smithi148 bash[79819]: cluster 2024-03-07T03:49:13.235081+0000 mgr.y (mgr.34101) 1058 : cluster 0 pgmap v1690: 105 pgs: 105 active+clean; 682 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 39 MiB/s wr, 352 op/s 2024-03-07T03:49:14.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:14 smithi148 bash[82222]: cluster 2024-03-07T03:49:13.014390+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2024-03-07T03:49:14.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:14 smithi148 bash[82222]: cluster 2024-03-07T03:49:13.235081+0000 mgr.y (mgr.34101) 1058 : cluster 0 pgmap v1690: 105 pgs: 105 active+clean; 682 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 39 MiB/s wr, 352 op/s 2024-03-07T03:49:14.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:14 smithi195 bash[66542]: cluster 2024-03-07T03:49:13.014390+0000 mon.a (mon.0) 1851 : cluster 0 osdmap e1013: 8 total, 8 up, 8 in 2024-03-07T03:49:14.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:14 smithi195 bash[66542]: cluster 2024-03-07T03:49:13.235081+0000 mgr.y (mgr.34101) 1058 : cluster 0 pgmap v1690: 105 pgs: 105 active+clean; 682 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 226 KiB/s rd, 39 MiB/s wr, 352 op/s 2024-03-07T03:49:15.128 INFO:tasks.workunit.client.0.smithi148.stdout: i=17 exists [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,410052091~3098836] wrote [41065687~5569033,57965719~1995106,73665886~6014867,179913275~579688,252711974~4792318,264307050~3516640,277702863~5379502,300469008~3708351,349211446~4192303,410052091~3098836] 2024-03-07T03:49:15.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:15 smithi148 bash[79819]: cluster 2024-03-07T03:49:14.028034+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2024-03-07T03:49:15.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:15 smithi148 bash[82222]: cluster 2024-03-07T03:49:14.028034+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2024-03-07T03:49:15.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:15 smithi195 bash[66542]: cluster 2024-03-07T03:49:14.028034+0000 mon.a (mon.0) 1852 : cluster 0 osdmap e1014: 8 total, 8 up, 8 in 2024-03-07T03:49:16.157 INFO:tasks.workunit.client.0.smithi148.stdout: i=18 exists [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,96448457~6460333,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] wrote [112259179~97462,136310253~353929,200312889~3020170,234565146~5173238,274480103~1473964,282881200~5421395,303919958~5634898,321502491~1021206,343434520~2214523,409873736~6085731] 2024-03-07T03:49:16.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:16 smithi148 bash[79819]: cluster 2024-03-07T03:49:15.029143+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2024-03-07T03:49:16.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:16 smithi148 bash[79819]: cluster 2024-03-07T03:49:15.235917+0000 mgr.y (mgr.34101) 1059 : cluster 0 pgmap v1693: 105 pgs: 105 active+clean; 718 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 220 KiB/s rd, 39 MiB/s wr, 341 op/s 2024-03-07T03:49:16.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:16 smithi148 bash[82222]: cluster 2024-03-07T03:49:15.029143+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2024-03-07T03:49:16.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:16 smithi148 bash[82222]: cluster 2024-03-07T03:49:15.235917+0000 mgr.y (mgr.34101) 1059 : cluster 0 pgmap v1693: 105 pgs: 105 active+clean; 718 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 220 KiB/s rd, 39 MiB/s wr, 341 op/s 2024-03-07T03:49:16.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:16 smithi195 bash[66542]: cluster 2024-03-07T03:49:15.029143+0000 mon.a (mon.0) 1853 : cluster 0 osdmap e1015: 8 total, 8 up, 8 in 2024-03-07T03:49:16.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:16 smithi195 bash[66542]: cluster 2024-03-07T03:49:15.235917+0000 mgr.y (mgr.34101) 1059 : cluster 0 pgmap v1693: 105 pgs: 105 active+clean; 718 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 220 KiB/s rd, 39 MiB/s wr, 341 op/s 2024-03-07T03:49:17.113 INFO:tasks.workunit.client.0.smithi148.stdout: i=19 exists [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,96448457~7805481,106634891~27679,107640648~1371632,111365223~15196568,131400499~24432036,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] wrote [1002676~3282513,10855115~389240,16821258~2887116,102509939~1743999,137114000~4310637,224437494~4921290,273147935~1987249,282591780~1737980,322725949~3038941,348686973~2943634] 2024-03-07T03:49:17.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:17 smithi148 bash[79819]: cluster 2024-03-07T03:49:16.047207+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2024-03-07T03:49:17.300 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:49:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:49:17] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-03-07T03:49:17.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:17 smithi148 bash[82222]: cluster 2024-03-07T03:49:16.047207+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2024-03-07T03:49:17.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:17 smithi195 bash[66542]: cluster 2024-03-07T03:49:16.047207+0000 mon.a (mon.0) 1854 : cluster 0 osdmap e1016: 8 total, 8 up, 8 in 2024-03-07T03:49:18.093 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 1 diff [26837245~5573006,50347152~272229,76104975~5291461,146288090~1790199,176574882~434201,230220240~7486763,241323653~2754980,264509899~7106289,306536384~4114836,356816538~6725480] 2024-03-07T03:49:18.093 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [26837245~5573006,50347152~272229,76104975~5291461,146288090~1790199,176574882~434201,230220240~7486763,241323653~2754980,264509899~7106289,306536384~4114836,356816538~6725480] 2024-03-07T03:49:18.146 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [25165824~7244427,50331648~287733,75497472~5898964,146288090~1790199,176160768~848315,226492416~11214587,239075328~5003305,264241152~7375036,306184192~4467028,356515840~7026178] 2024-03-07T03:49:18.147 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 2 diff [6453699~2873285,17913684~4306597,22356670~10053581,50347152~272229,76104975~5291461,135946328~6583779,146288090~1790199,176574882~434201,230220240~7486763,241323653~2754980,264509899~7106289,306536384~4114836,312442734~5263519,322318926~5618689,356612098~7305062,404168647~1959084] 2024-03-07T03:49:18.147 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [6453699~2873285,17913684~4306597,22356670~10053581,50347152~272229,76104975~5291461,135946328~6583779,146288090~1790199,176574882~434201,230220240~7486763,241323653~2754980,264509899~7106289,306536384~4114836,312442734~5263519,322318926~5618689,356612098~7305062,404168647~1959084] 2024-03-07T03:49:18.163 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~5132680,16777216~15633035,50331648~287733,75497472~5898964,135946328~6583779,146288090~1790199,176160768~848315,226492416~11214587,239075328~5003305,264241152~7375036,306184192~4467028,312442734~5263519,320748326~7189289,356515840~7401320,402653184~3474547] 2024-03-07T03:49:18.163 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 3 diff [3227891~6118297,17913684~4306597,22356670~10053581,50347152~272229,76104975~5291461,87450726~3021868,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,146288090~1790199,176574882~434201,182263066~5648763,230220240~7486763,241323653~2754980,264509899~7106289,306536384~4114836,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,404168647~1959084] 2024-03-07T03:49:18.163 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3227891~6118297,17913684~4306597,22356670~10053581,50347152~272229,76104975~5291461,87450726~3021868,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,146288090~1790199,176574882~434201,182263066~5648763,230220240~7486763,241323653~2754980,264509899~7106289,306536384~4114836,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,404168647~1959084] 2024-03-07T03:49:18.178 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,16777216~15633035,50331648~287733,75497472~5898964,83886080~6586514,109051904~5449665,115063698~1259632,121634816~4926975,135946328~6583779,146288090~1790199,176160768~848315,180355072~7556757,226492416~11214587,239075328~5003305,264241152~7375036,306184192~4467028,312442734~5263519,320748326~7189289,343932928~9539975,356515840~7401320,364904448~14548624,402653184~3474547] 2024-03-07T03:49:18.178 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 4 diff [3227891~6118297,17913684~4306597,22356670~10053581,47457196~10921189,70050257~2762559,75213635~6182801,86189502~5165955,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,146288090~8293755,176574882~434201,178456707~9455122,230220240~8131429,241323653~2754980,264509899~9916446,306536384~4114836,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.178 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3227891~6118297,17913684~4306597,22356670~10053581,47457196~10921189,70050257~2762559,75213635~6182801,86189502~5165955,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,146288090~8293755,176574882~434201,178456707~9455122,230220240~8131429,241323653~2754980,264509899~9916446,306536384~4114836,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.192 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,16777216~15633035,46137344~12241041,67108864~14287572,83886080~7469377,109051904~5449665,115063698~1259632,121634816~4926975,135946328~6583779,146288090~8293755,176160768~11751061,226492416~11859253,239075328~5003305,264241152~10185193,306184192~4467028,312442734~5263519,320748326~7189289,343932928~9539975,356515840~7401320,364904448~17493148,402653184~3474547] 2024-03-07T03:49:18.192 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 5 diff [1668238~7677950,17913684~4306597,22356670~10053581,47457196~10921189,70050257~11346179,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146288090~8293755,176574882~434201,178456707~9455122,230220240~8131429,241323653~2754980,264509899~9916446,293107900~4881608,306536384~4114836,312442734~5263519,322318926~5618689,331579789~6133207,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.192 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,17913684~4306597,22356670~10053581,47457196~10921189,70050257~11346179,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146288090~8293755,176574882~434201,178456707~9455122,230220240~8131429,241323653~2754980,264509899~9916446,293107900~4881608,306536384~4114836,312442734~5263519,322318926~5618689,331579789~6133207,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.203 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,16777216~15633035,46137344~12241041,67108864~14287572,83886080~7469377,101146541~1069076,109051904~17509887,131400499~11129608,146288090~8293755,176160768~11751061,226492416~11859253,239075328~5003305,264241152~10185193,289406976~8582532,306184192~4467028,312442734~5263519,320748326~7189289,331350016~6362980,343932928~9539975,356515840~7401320,364904448~17493148,402653184~3474547] 2024-03-07T03:49:18.203 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 6 diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~5634426,47457196~10921189,70050257~11346179,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146288090~8293755,176574882~434201,178456707~9455122,217208288~1229660,230220240~8131429,241323653~2754980,260950876~13475469,293107900~4881608,306536384~11169869,322318926~5618689,331579789~6914742,345377430~8095473,356612098~7305062,367989998~11463074,380459900~1937696,388314730~1562479,404168647~1959084] 2024-03-07T03:49:18.204 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~5634426,47457196~10921189,70050257~11346179,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146288090~8293755,176574882~434201,178456707~9455122,217208288~1229660,230220240~8131429,241323653~2754980,260950876~13475469,293107900~4881608,306536384~11169869,322318926~5618689,331579789~6914742,345377430~8095473,356612098~7305062,367989998~11463074,380459900~1937696,388314730~1562479,404168647~1959084] 2024-03-07T03:49:18.214 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~19827339,37748736~6621320,46137344~12241041,67108864~14287572,83886080~7469377,101146541~1069076,109051904~17509887,131400499~11129608,146288090~8293755,176160768~11751061,213909504~4528444,226492416~11859253,239075328~5003305,260046848~14379497,289406976~8582532,306184192~11522061,320748326~7189289,331350016~7144515,343932928~9539975,356515840~7401320,364904448~17493148,385875968~4001241,402653184~3474547] 2024-03-07T03:49:18.214 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 7 diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,70050257~11346179,86189502~5165955,101146541~1069076,111365223~15196568,131400499~12002268,146288090~8293755,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~4186570,230220240~8131429,241323653~2754980,260950876~13475469,286966749~11022759,306536384~11169869,322318926~5618689,331579789~6914742,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.214 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,70050257~11346179,86189502~5165955,101146541~1069076,111365223~15196568,131400499~12002268,146288090~8293755,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~4186570,230220240~8131429,241323653~2754980,260950876~13475469,286966749~11022759,306536384~11169869,322318926~5618689,331579789~6914742,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.224 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~19827339,37748736~23656516,67108864~14287572,83886080~7469377,101146541~1069076,109051904~17509887,131400499~12002268,146288090~8293755,171966464~1816793,176160768~11751061,213909504~10184575,226492416~11859253,239075328~5003305,260046848~14379497,285212672~12776836,306184192~11522061,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356515840~7401320,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.224 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 8 diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,70050257~11346179,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,146288090~8293755,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~4186570,230220240~13858393,260950876~13475469,283198497~14791011,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.224 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,70050257~11346179,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,146288090~8293755,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~4186570,230220240~13858393,260950876~13475469,283198497~14791011,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.233 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~19827339,37748736~23656516,67108864~14287572,83886080~7469377,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,146288090~8293755,171966464~1816793,176160768~11751061,213909504~10184575,226492416~17586217,260046848~14379497,281018368~16971140,306184192~12605147,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356515840~7401320,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.233 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 9 diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,68844351~12552085,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~8293755,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~8387335,230220240~13858393,258627946~17857475,278150166~19839342,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.233 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~10053581,38735630~7654854,47457196~13948056,68844351~12552085,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~8293755,172622639~1160618,176574882~434201,178456707~9455122,217208288~1229660,219907509~8387335,230220240~13858393,258627946~17857475,278150166~19839342,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.242 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~19827339,37748736~23656516,67108864~14287572,83886080~7469377,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~8293755,171966464~1816793,176160768~11751061,213909504~30169129,255852544~20632877,276824064~21165444,306184192~12605147,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356515840~7401320,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.242 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 10 diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~8293755,172622639~1160618,175143458~13391092,217208288~1229660,219907509~8387335,230220240~13858393,248347320~4168563,258627946~18162098,278150166~19839342,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.242 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~8293755,172622639~1160618,175143458~13391092,217208288~1229660,219907509~8387335,230220240~13858393,248347320~4168563,258627946~18162098,278150166~19839342,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.250 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~23656516,67108864~24246593,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~8293755,171966464~16568086,213909504~30169129,248347320~4168563,255852544~20937500,276824064~21165444,306184192~12605147,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356515840~7401320,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.250 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 11 diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,172622639~1160618,175143458~13391092,217208288~1229660,219907509~24171124,248347320~4790785,258627946~18162098,278150166~19839342,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.250 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,172622639~1160618,175143458~13391092,217208288~1229660,219907509~24171124,248347320~4790785,258627946~18162098,278150166~19839342,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.257 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~9544445,171966464~16568086,213909504~30169129,248347320~4790785,255852544~20937500,276824064~21165444,306184192~12605147,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356515840~25881756,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.257 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 12 diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,172622639~1160618,175143458~13391092,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394784133~6342267,404168647~1959084] 2024-03-07T03:49:18.257 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,172622639~1160618,175143458~13391092,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~12252955,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394784133~6342267,404168647~1959084] 2024-03-07T03:49:18.264 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~9544445,171966464~16568086,213909504~34393346,248347320~4790785,255852544~20937500,276824064~25748982,306184192~12605147,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.264 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 13 diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~3711001,172622639~1160618,175143458~13391092,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~12252955,322318926~5618689,330544483~7950048,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.265 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~3711001,172622639~1160618,175143458~13391092,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~12252955,322318926~5618689,330544483~7950048,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.272 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~16408560,171966464~16568086,213909504~34393346,248347320~4790785,255852544~20937500,276824064~25748982,306184192~12605147,320748326~17746205,338641176~1731708,343932928~9539975,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.272 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 14 diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~3711001,172622639~1160618,175143458~13391092,207908122~5517064,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~31958147,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.272 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~3711001,172622639~1160618,175143458~13391092,207908122~5517064,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~31958147,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.278 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~16408560,171966464~16568086,205520896~7904290,213909504~34393346,248347320~4790785,255852544~20937500,276824064~25748982,306184192~32310339,338641176~1731708,343932928~9539975,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.279 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 15 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~3711001,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.279 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~3711001,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.285 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~16408560,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~4790785,255852544~20937500,276824064~25748982,306184192~34188692,343932928~9539975,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.285 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 16 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.285 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.292 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~4790785,255852544~20937500,276824064~25748982,306184192~34188692,343932928~9539975,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.292 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 17 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,410052091~3098836] 2024-03-07T03:49:18.292 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,306536384~33836500,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,410052091~3098836] 2024-03-07T03:49:18.299 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~27353295,306184192~34188692,343932928~9539975,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~6303439] 2024-03-07T03:49:18.299 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 18 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.299 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.306 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:18.306 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 19 diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.306 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.312 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:18.312 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 2 diff [6453699~2873285,17913684~4306597,22356670~5955932,135946328~6583779,312442734~5263519,322318926~5618689,356612098~7305062,404168647~1959084] 2024-03-07T03:49:18.312 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [6453699~2873285,17913684~4306597,22356670~5955932,135946328~6583779,312442734~5263519,322318926~5618689,356612098~7305062,404168647~1959084] 2024-03-07T03:49:18.319 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~5132680,16777216~11535386,135946328~6583779,312442734~5263519,320748326~7189289,356612098~7305062,402653184~3474547] 2024-03-07T03:49:18.319 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 3 diff [3227891~6118297,17913684~4306597,22356670~5955932,87450726~3021868,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,182263066~5648763,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,404168647~1959084] 2024-03-07T03:49:18.319 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3227891~6118297,17913684~4306597,22356670~5955932,87450726~3021868,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,182263066~5648763,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,404168647~1959084] 2024-03-07T03:49:18.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:18 smithi195 bash[66542]: cluster 2024-03-07T03:49:17.049074+0000 mon.a (mon.0) 1855 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2024-03-07T03:49:18.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:18 smithi195 bash[66542]: cluster 2024-03-07T03:49:17.236611+0000 mgr.y (mgr.34101) 1060 : cluster 0 pgmap v1696: 105 pgs: 105 active+clean; 760 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 18 MiB/s wr, 185 op/s 2024-03-07T03:49:18.327 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,16777216~11535386,83886080~6586514,109051904~5449665,115063698~1259632,121634816~4926975,135946328~6583779,180355072~7556757,312442734~5263519,320748326~7189289,343932928~9539975,356612098~7305062,364904448~14548624,402653184~3474547] 2024-03-07T03:49:18.327 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 4 diff [3227891~6118297,17913684~4306597,22356670~5955932,47457196~10921189,70050257~2762559,75213635~1919358,86189502~5165955,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.327 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3227891~6118297,17913684~4306597,22356670~5955932,47457196~10921189,70050257~2762559,75213635~1919358,86189502~5165955,111365223~3136346,115063698~1259632,124635782~1926009,135946328~6583779,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,312442734~5263519,322318926~5618689,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.334 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,16777216~11535386,46137344~12241041,67108864~10024129,83886080~7469377,109051904~5449665,115063698~1259632,121634816~4926975,135946328~6583779,146733854~7847991,177009083~10902746,233433683~4917986,266839104~7587241,312442734~5263519,320748326~7189289,343932928~9539975,356612098~7305062,364904448~17493148,402653184~3474547] 2024-03-07T03:49:18.334 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 5 diff [1668238~7677950,17913684~4306597,22356670~5955932,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,293107900~4881608,312442734~5263519,322318926~5618689,331579789~6133207,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.334 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,17913684~4306597,22356670~5955932,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,293107900~4881608,312442734~5263519,322318926~5618689,331579789~6133207,346627350~6845553,356612098~7305062,367989998~11463074,380459900~1937696,404168647~1959084] 2024-03-07T03:49:18.340 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,16777216~11535386,46137344~12241041,67108864~11360994,83886080~7469377,101146541~1069076,109051904~17509887,131400499~11129608,146733854~7847991,177009083~10902746,233433683~4917986,266839104~7587241,289406976~8582532,312442734~5263519,320748326~7189289,331350016~6362980,343932928~9539975,356612098~7305062,364904448~17493148,402653184~3474547] 2024-03-07T03:49:18.340 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 6 diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~5634426,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146733854~7847991,178456707~9455122,217208288~1229660,233433683~4917986,260950876~13475469,293107900~4881608,308958519~8747734,322318926~5618689,331579789~6914742,345377430~8095473,356612098~7305062,367989998~11463074,380459900~1937696,388314730~1562479,404168647~1959084] 2024-03-07T03:49:18.341 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~5634426,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~11129608,146733854~7847991,178456707~9455122,217208288~1229660,233433683~4917986,260950876~13475469,293107900~4881608,308958519~8747734,322318926~5618689,331579789~6914742,345377430~8095473,356612098~7305062,367989998~11463074,380459900~1937696,388314730~1562479,404168647~1959084] 2024-03-07T03:49:18.347 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~15729690,37748736~6621320,46137344~12241041,67108864~11360994,83886080~7469377,101146541~1069076,109051904~17509887,131400499~11129608,146733854~7847991,177009083~10902746,213909504~4528444,233433683~4917986,260046848~14379497,289406976~8582532,308958519~8747734,320748326~7189289,331350016~7144515,343932928~9539975,356612098~7305062,364904448~17493148,385875968~4001241,402653184~3474547] 2024-03-07T03:49:18.347 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 7 diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~7654854,47457196~13948056,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~4917986,260950876~13475469,286966749~11022759,308958519~8747734,322318926~5618689,331579789~6914742,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.347 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~7654854,47457196~13948056,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~4917986,260950876~13475469,286966749~11022759,308958519~8747734,322318926~5618689,331579789~6914742,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.354 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~15729690,37748736~23656516,67108864~11360994,83886080~7469377,101146541~1069076,109051904~17509887,131400499~12002268,146733854~7847991,171966464~1816793,177009083~10902746,213909504~10184575,233433683~4917986,260046848~14379497,285212672~12776836,308958519~8747734,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356612098~7305062,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.354 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 8 diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~7654854,47457196~13948056,70050257~9078206,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~8254427,260950876~13475469,283198497~14791011,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.354 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~7654854,47457196~13948056,70050257~9078206,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~8254427,260950876~13475469,283198497~14791011,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.361 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~15729690,37748736~23656516,67108864~12019599,83886080~7469377,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,146733854~7847991,171966464~1816793,177009083~10902746,213909504~10184575,233433683~8254427,260046848~14379497,281018368~16971140,308958519~9830820,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356612098~7305062,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.361 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 9 diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~7654854,47457196~13948056,68844351~10284112,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~8387335,233433683~8254427,258627946~17857475,278150166~19839342,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.361 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~5955932,38735630~7654854,47457196~13948056,68844351~10284112,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~8387335,233433683~8254427,258627946~17857475,278150166~19839342,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.368 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~15729690,37748736~23656516,67108864~12019599,83886080~7469377,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~7847991,171966464~1816793,177009083~10902746,213909504~14385340,233433683~8254427,255852544~20632877,276824064~21165444,308958519~9830820,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356612098~7305062,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.368 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 10 diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,175143458~13391092,217208288~1229660,219907509~8387335,233433683~8254427,248347320~4168563,258627946~18162098,278150166~19839342,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.368 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,175143458~13391092,217208288~1229660,219907509~8387335,233433683~8254427,248347320~4168563,258627946~18162098,278150166~19839342,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~7305062,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.375 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~23656516,67108864~24246593,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~7847991,171966464~16568086,213909504~14385340,233433683~8254427,248347320~4168563,255852544~20937500,276824064~21165444,308958519~9830820,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356612098~7305062,364904448~17493148,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.375 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 11 diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12085719,233433683~8254427,248347320~4790785,258627946~18162098,278150166~19839342,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.375 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5901273,22356670~13193518,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12085719,233433683~8254427,248347320~4790785,258627946~18162098,278150166~19839342,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,388314730~1562479,394784133~2473510,404168647~1959084] 2024-03-07T03:49:18.382 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~9098681,171966464~16568086,213909504~18083724,233433683~8254427,248347320~4790785,255852544~20937500,276824064~21165444,308958519~9830820,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356612098~25785498,385875968~4001241,394264576~2993067,402653184~3474547] 2024-03-07T03:49:18.382 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 12 diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394784133~6342267,404168647~1959084] 2024-03-07T03:49:18.382 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322318926~5618689,331500133~6994398,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394784133~6342267,404168647~1959084] 2024-03-07T03:49:18.389 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~9098681,171966464~16568086,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~9830820,320748326~7189289,331350016~7144515,338641176~1731708,343932928~9539975,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.389 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 13 diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322318926~5618689,330544483~7950048,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.389 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322318926~5618689,330544483~7950048,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.395 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~15962796,171966464~16568086,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~9830820,320748326~17746205,338641176~1731708,343932928~9539975,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.396 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 14 diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.396 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~19231180,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.403 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~22967276,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~15962796,171966464~16568086,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~29536012,338641176~1731708,343932928~9539975,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.404 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 15 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.404 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.416 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~15962796,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~31414365,343932928~9539975,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.416 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 16 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.416 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084] 2024-03-07T03:49:18.422 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~31414365,343932928~9539975,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547] 2024-03-07T03:49:18.422 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 17 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,410052091~3098836] 2024-03-07T03:49:18.423 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8095473,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,410052091~3098836] 2024-03-07T03:49:18.429 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~27353295,308958519~31414365,343932928~9539975,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~6303439] 2024-03-07T03:49:18.429 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 18 diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.429 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.435 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:18.436 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 19 diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.436 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:18.442 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:18.442 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 3 diff [3227891~6118297,87450726~3021868,111365223~3136346,115063698~1259632,124635782~1926009,182263066~5648763,346627350~6845553,367989998~11463074] 2024-03-07T03:49:18.442 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3227891~6118297,87450726~3021868,111365223~3136346,115063698~1259632,124635782~1926009,182263066~5648763,346627350~6845553,367989998~11463074] 2024-03-07T03:49:18.448 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,83886080~6586514,109051904~5449665,115063698~1259632,121634816~4926975,180355072~7556757,343932928~9539975,364904448~14548624] 2024-03-07T03:49:18.449 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 4 diff [3227891~6118297,47457196~10921189,70050257~2762559,75213635~1919358,86189502~5165955,111365223~3136346,115063698~1259632,124635782~1926009,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,346627350~6845553,367989998~11463074,380459900~1937696] 2024-03-07T03:49:18.449 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3227891~6118297,47457196~10921189,70050257~2762559,75213635~1919358,86189502~5165955,111365223~3136346,115063698~1259632,124635782~1926009,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,346627350~6845553,367989998~11463074,380459900~1937696] 2024-03-07T03:49:18.455 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,46137344~12241041,67108864~10024129,83886080~7469377,109051904~5449665,115063698~1259632,121634816~4926975,146733854~7847991,177009083~10902746,233433683~4917986,266839104~7587241,343932928~9539975,364904448~17493148] 2024-03-07T03:49:18.456 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 5 diff [1668238~7677950,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~8153873,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,293107900~4881608,331579789~6133207,346627350~6845553,367989998~11463074,380459900~1937696] 2024-03-07T03:49:18.456 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~8153873,146733854~7847991,178456707~9455122,233433683~4917986,266839104~7587241,293107900~4881608,331579789~6133207,346627350~6845553,367989998~11463074,380459900~1937696] 2024-03-07T03:49:18.462 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,46137344~12241041,67108864~11360994,83886080~7469377,101146541~1069076,109051904~17509887,131400499~8153873,146733854~7847991,177009083~10902746,233433683~4917986,266839104~7587241,289406976~8582532,331350016~6362980,343932928~9539975,364904448~17493148] 2024-03-07T03:49:18.462 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 6 diff [1668238~7677950,16319008~5418293,38735630~5634426,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~8153873,146733854~7847991,178456707~9455122,217208288~1229660,233433683~4917986,260950876~13475469,293107900~4881608,308958519~7144889,331579789~6914742,345377430~8095473,367989998~11463074,380459900~1937696,388314730~1562479] 2024-03-07T03:49:18.462 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5418293,38735630~5634426,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~8153873,146733854~7847991,178456707~9455122,217208288~1229660,233433683~4917986,260950876~13475469,293107900~4881608,308958519~7144889,331579789~6914742,345377430~8095473,367989998~11463074,380459900~1937696,388314730~1562479] 2024-03-07T03:49:18.469 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~9154389,37748736~6621320,46137344~12241041,67108864~11360994,83886080~7469377,101146541~1069076,109051904~17509887,131400499~8153873,146733854~7847991,177009083~10902746,213909504~4528444,233433683~4917986,260046848~14379497,289406976~8582532,308958519~7144889,331350016~7144515,343932928~9539975,364904448~17493148,385875968~4001241] 2024-03-07T03:49:18.469 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 7 diff [1668238~7677950,16319008~5421244,38735630~7654854,47457196~13948056,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~4917986,260950876~13475469,286966749~11022759,308958519~7144889,331579789~6914742,338641176~1731708,345377430~8095473,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.469 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5421244,38735630~7654854,47457196~13948056,70050257~8419601,86189502~5165955,101146541~1069076,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~4917986,260950876~13475469,286966749~11022759,308958519~7144889,331579789~6914742,338641176~1731708,345377430~8095473,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.476 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~9157340,37748736~23656516,67108864~11360994,83886080~7469377,101146541~1069076,109051904~17509887,131400499~12002268,146733854~7847991,171966464~1816793,177009083~10902746,213909504~10184575,233433683~4917986,260046848~14379497,285212672~12776836,308958519~7144889,331350016~7144515,338641176~1731708,343932928~9539975,364904448~17493148,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.476 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 8 diff [1668238~7677950,16319008~5421244,38735630~7654854,47457196~13948056,70050257~9078206,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~8254427,260950876~13475469,283198497~14791011,308958519~9830820,331500133~6994398,338641176~1731708,345377430~8095473,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.476 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5421244,38735630~7654854,47457196~13948056,70050257~9078206,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~4186570,233433683~8254427,260950876~13475469,283198497~14791011,308958519~9830820,331500133~6994398,338641176~1731708,345377430~8095473,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.483 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~9157340,37748736~23656516,67108864~12019599,83886080~7469377,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,146733854~7847991,171966464~1816793,177009083~10902746,213909504~10184575,233433683~8254427,260046848~14379497,281018368~16971140,308958519~9830820,331350016~7144515,338641176~1731708,343932928~9539975,364904448~17493148,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.483 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 9 diff [1668238~7677950,16319008~5421244,38735630~7654854,47457196~13948056,68844351~10284112,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~8387335,233433683~8254427,258627946~17857475,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.483 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5421244,38735630~7654854,47457196~13948056,68844351~10284112,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,178456707~9455122,217208288~1229660,219907509~8387335,233433683~8254427,258627946~17857475,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.489 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~9157340,37748736~23656516,67108864~12019599,83886080~7469377,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~7847991,171966464~1816793,177009083~10902746,213909504~14385340,233433683~8254427,255852544~20632877,276824064~21165444,308958519~9830820,331350016~7144515,338641176~1731708,343932928~9539975,357034236~4493146,364904448~17493148,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.489 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 10 diff [1668238~7677950,16319008~5421244,27243122~8307066,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,175143458~13391092,217208288~1229660,219907509~8387335,233433683~8254427,248347320~4168563,258627946~18162098,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.489 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5421244,27243122~8307066,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,175143458~13391092,217208288~1229660,219907509~8387335,233433683~8254427,248347320~4168563,258627946~18162098,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,365771831~13681241,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.496 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~9157340,27243122~8307066,37748736~23656516,67108864~24246593,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~7847991,171966464~16568086,213909504~14385340,233433683~8254427,248347320~4168563,255852544~20937500,276824064~21165444,308958519~9830820,331350016~7144515,338641176~1731708,343932928~9539975,357034236~4493146,364904448~17493148,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.497 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 11 diff [1668238~7677950,16319008~5421244,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12085719,233433683~8254427,248347320~4790785,258627946~18162098,278150166~19839342,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,362085616~17367456,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.497 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~5421244,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12085719,233433683~8254427,248347320~4790785,258627946~18162098,278150166~19839342,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,362085616~17367456,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.503 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~9157340,27243122~8307066,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~9098681,171966464~16568086,213909504~18083724,233433683~8254427,248347320~4790785,255852544~20937500,276824064~21165444,308958519~9830820,322402909~5512743,331350016~7144515,338641176~1731708,343932928~9539975,357034236~4493146,362085616~20311980,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.503 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 12 diff [1668238~7677950,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,362085616~17367456,380459900~1937696,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.503 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~8095473,357034236~4493146,362085616~17367456,380459900~1937696,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.510 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~12098108,27243122~8307066,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~9098681,171966464~16568086,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~9830820,322402909~5512743,331350016~7144515,338641176~1731708,343932928~9539975,357034236~4493146,362085616~20311980,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.510 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 13 diff [1668238~7677950,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~8095473,357034236~4493146,362085616~17367456,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.510 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~8095473,357034236~4493146,362085616~17367456,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.517 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~12098108,27243122~8307066,37748736~24978295,62914560~28440897,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~15962796,171966464~16568086,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~9830820,322402909~5512743,327937615~10556916,338641176~1731708,343932928~9539975,357034236~4493146,362085616~20311980,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.517 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 14 diff [1668238~7677950,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.517 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.523 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~12098108,27243122~8307066,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~15962796,171966464~16568086,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~29536012,338641176~1731708,343932928~9539975,357034236~25363360,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.523 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 15 diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.523 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.530 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~15962796,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~31414365,343932928~9539975,357034236~25363360,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.530 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 16 diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.530 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.536 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~31414365,343932928~9539975,357034236~25363360,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.536 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 17 diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.536 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8095473,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.543 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~27353295,308958519~31414365,343932928~9539975,357034236~25363360,385875968~8071022,394264576~6861824,406847488~6303439] 2024-03-07T03:49:18.543 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 18 diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.543 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~7677950,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:18 smithi148 bash[79819]: cluster 2024-03-07T03:49:17.049074+0000 mon.a (mon.0) 1855 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2024-03-07T03:49:18.549 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:18 smithi148 bash[79819]: cluster 2024-03-07T03:49:17.236611+0000 mgr.y (mgr.34101) 1060 : cluster 0 pgmap v1696: 105 pgs: 105 active+clean; 760 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 18 MiB/s wr, 185 op/s 2024-03-07T03:49:18.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:18 smithi148 bash[82222]: cluster 2024-03-07T03:49:17.049074+0000 mon.a (mon.0) 1855 : cluster 0 osdmap e1017: 8 total, 8 up, 8 in 2024-03-07T03:49:18.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:18 smithi148 bash[82222]: cluster 2024-03-07T03:49:17.236611+0000 mgr.y (mgr.34101) 1060 : cluster 0 pgmap v1696: 105 pgs: 105 active+clean; 760 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 110 KiB/s rd, 18 MiB/s wr, 185 op/s 2024-03-07T03:49:18.550 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~9346188,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.550 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 19 diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.550 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.556 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.557 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 4 diff [47457196~10921189,70050257~2762559,75213635~1919358,86189502~5165955,146733854~7847991,178456707~4628361,233433683~4917986,266839104~7587241,380459900~1937696] 2024-03-07T03:49:18.557 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [47457196~10921189,70050257~2762559,75213635~1919358,86189502~5165955,146733854~7847991,178456707~4628361,233433683~4917986,266839104~7587241,380459900~1937696] 2024-03-07T03:49:18.563 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [46137344~12241041,67108864~10024129,86189502~5165955,146733854~7847991,177009083~6075985,233433683~4917986,266839104~7587241,379453072~2944524] 2024-03-07T03:49:18.563 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 5 diff [1668238~4944408,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,114140049~10701635,131400499~8153873,146733854~7847991,178456707~4628361,233433683~4917986,266839104~7587241,293107900~4881608,331579789~6133207,349128013~3657743,380459900~1937696] 2024-03-07T03:49:18.563 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~4944408,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,114140049~10701635,131400499~8153873,146733854~7847991,178456707~4628361,233433683~4917986,266839104~7587241,293107900~4881608,331579789~6133207,349128013~3657743,380459900~1937696] 2024-03-07T03:49:18.570 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~4944408,46137344~12241041,67108864~11360994,86189502~5165955,101146541~1069076,114140049~10701635,131400499~8153873,146733854~7847991,177009083~6075985,233433683~4917986,266839104~7587241,289406976~8582532,331350016~6362980,349128013~3657743,379453072~2944524] 2024-03-07T03:49:18.570 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 6 diff [1668238~6417205,16319008~5418293,38735630~5634426,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,114140049~10701635,131400499~8153873,146733854~7847991,178456707~4628361,217208288~1229660,233433683~4917986,260950876~13475469,293107900~4881608,308958519~7144889,331579789~6914742,345377430~7491289,380459900~1937696,388314730~1562479] 2024-03-07T03:49:18.570 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5418293,38735630~5634426,47457196~10921189,70050257~8419601,86189502~5165955,101146541~1069076,114140049~10701635,131400499~8153873,146733854~7847991,178456707~4628361,217208288~1229660,233433683~4917986,260950876~13475469,293107900~4881608,308958519~7144889,331579789~6914742,345377430~7491289,380459900~1937696,388314730~1562479] 2024-03-07T03:49:18.576 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9154389,37748736~6621320,46137344~12241041,67108864~11360994,86189502~5165955,101146541~1069076,114140049~10701635,131400499~8153873,146733854~7847991,177009083~6075985,213909504~4528444,233433683~4917986,260046848~14379497,289406976~8582532,308958519~7144889,331350016~7144515,345377430~7491289,379453072~2944524,385875968~4001241] 2024-03-07T03:49:18.577 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 7 diff [1668238~6417205,16319008~5421244,38735630~7654854,47457196~13948056,70050257~8419601,86189502~5165955,101146541~1069076,114140049~10701635,131400499~12002268,146733854~7847991,172622639~1160618,178456707~4628361,217208288~1229660,219907509~4186570,233433683~4917986,260950876~13475469,286966749~11022759,308958519~7144889,331579789~6914742,338641176~1731708,345377430~7491289,365771831~4427582,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.577 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,38735630~7654854,47457196~13948056,70050257~8419601,86189502~5165955,101146541~1069076,114140049~10701635,131400499~12002268,146733854~7847991,172622639~1160618,178456707~4628361,217208288~1229660,219907509~4186570,233433683~4917986,260950876~13475469,286966749~11022759,308958519~7144889,331579789~6914742,338641176~1731708,345377430~7491289,365771831~4427582,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.583 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,37748736~23656516,67108864~11360994,86189502~5165955,101146541~1069076,114140049~10701635,131400499~12002268,146733854~7847991,171966464~1816793,177009083~6075985,213909504~10184575,233433683~4917986,260046848~14379497,285212672~12776836,308958519~7144889,331350016~7144515,338641176~1731708,345377430~7491289,365771831~4427582,379453072~2944524,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.583 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 8 diff [1668238~6417205,16319008~5421244,38735630~7654854,47457196~13948056,70050257~9078206,86189502~5165955,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,146733854~7847991,172622639~1160618,178456707~4628361,217208288~1229660,219907509~4186570,233433683~8254427,260950876~13475469,283198497~14791011,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,365771831~9748242,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.583 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,38735630~7654854,47457196~13948056,70050257~9078206,86189502~5165955,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,146733854~7847991,172622639~1160618,178456707~4628361,217208288~1229660,219907509~4186570,233433683~8254427,260950876~13475469,283198497~14791011,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,365771831~9748242,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.589 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,37748736~23656516,67108864~12019599,86189502~5165955,101146541~1069076,106662570~2349710,114140049~10701635,131400499~12002268,146733854~7847991,171966464~1816793,177009083~6075985,213909504~10184575,233433683~8254427,260046848~14379497,281018368~16971140,308958519~9830820,331350016~7144515,338641176~1731708,345377430~7491289,365771831~9748242,379453072~2944524,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.589 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 9 diff [1668238~6417205,16319008~5421244,38735630~7654854,47457196~13948056,68844351~10284112,86189502~5165955,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,178456707~4628361,217208288~1229660,219907509~8387335,233433683~8254427,258627946~17857475,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.589 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,38735630~7654854,47457196~13948056,68844351~10284112,86189502~5165955,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,178456707~4628361,217208288~1229660,219907509~8387335,233433683~8254427,258627946~17857475,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.596 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,37748736~23656516,67108864~12019599,86189502~5165955,101146541~1069076,106662570~2349710,114140049~10701635,131400499~12002268,143458483~109788,146733854~7847991,171966464~1816793,177009083~6075985,213909504~14385340,233433683~8254427,255852544~20632877,276824064~21165444,308958519~9830820,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,379453072~2944524,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.596 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 10 diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,175143458~7941610,187362711~1171839,217208288~1229660,219907509~8387335,233433683~8254427,248347320~4168563,258627946~18162098,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.596 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,47457196~13948056,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~7847991,172622639~1160618,175143458~7941610,187362711~1171839,217208288~1229660,219907509~8387335,233433683~8254427,248347320~4168563,258627946~18162098,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.603 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,27243122~8307066,37748736~23656516,67108864~17906874,86189502~5165955,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~7847991,171966464~11118604,187362711~1171839,213909504~14385340,233433683~8254427,248347320~4168563,255852544~20937500,276824064~21165444,308958519~9830820,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,379453072~2944524,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.603 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 11 diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~7941610,187362711~1171839,217208288~1229660,219907509~12085719,233433683~8254427,248347320~4790785,258627946~18162098,278150166~19839342,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.603 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~7941610,187362711~1171839,217208288~1229660,219907509~12085719,233433683~8254427,248347320~4790785,258627946~18162098,278150166~19839342,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,380459900~1937696,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.611 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,27243122~8307066,37748736~24978295,62914560~22101178,86189502~5165955,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,171966464~11118604,187362711~1171839,213909504~18083724,233433683~8254427,248347320~4790785,255852544~20937500,276824064~21165444,308958519~9830820,322402909~5512743,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,379453072~2944524,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.611 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 12 diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~7941610,187362711~1171839,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,380459900~1937696,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.611 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,172622639~1160618,175143458~7941610,187362711~1171839,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,380459900~1937696,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.618 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~8307066,37748736~24978295,62914560~22101178,86189502~5165955,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,171966464~11118604,187362711~1171839,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~9830820,322402909~5512743,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.618 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 13 diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~9038177,187362711~1171839,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.618 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~9038177,187362711~1171839,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.624 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~8307066,37748736~24978295,62914560~22101178,86189502~5165955,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~15962796,171966464~12215171,187362711~1171839,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~9830820,322402909~5512743,327937615~10556916,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.624 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 14 diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~9038177,187362711~1171839,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.624 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~9038177,187362711~1171839,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.631 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~8307066,37748736~24978295,62914560~22101178,86189502~5165955,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~15962796,171966464~12215171,187362711~1171839,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.631 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 15 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.631 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~16171387,86189502~5165955,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~3711001,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.638 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~24978295,62914560~22101178,86189502~5165955,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~15962796,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.638 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 16 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.638 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~4790785,258627946~18162098,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.645 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~4790785,255852544~20937500,276824064~25748982,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.645 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 17 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.646 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.652 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~27353295,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~6303439] 2024-03-07T03:49:18.652 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 18 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.652 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.660 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.660 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 19 diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.660 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.667 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.667 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 5 diff [1668238~4944408,72347956~6121902,101146541~1069076,114140049~10701635,131400499~8153873,293107900~4881608,331579789~6133207,349128013~3657743] 2024-03-07T03:49:18.667 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~4944408,72347956~6121902,101146541~1069076,114140049~10701635,131400499~8153873,293107900~4881608,331579789~6133207,349128013~3657743] 2024-03-07T03:49:18.674 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~4944408,72347956~6121902,101146541~1069076,114140049~10701635,131400499~8153873,289406976~8582532,331350016~6362980,349128013~3657743] 2024-03-07T03:49:18.674 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 6 diff [1668238~6417205,16319008~5418293,38735630~5634426,72347956~6121902,101146541~1069076,114140049~10701635,131400499~8153873,217208288~1229660,260950876~7549629,293107900~4881608,308958519~7144889,331579789~6914742,345377430~7491289,388314730~1562479] 2024-03-07T03:49:18.674 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5418293,38735630~5634426,72347956~6121902,101146541~1069076,114140049~10701635,131400499~8153873,217208288~1229660,260950876~7549629,293107900~4881608,308958519~7144889,331579789~6914742,345377430~7491289,388314730~1562479] 2024-03-07T03:49:18.680 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9154389,37748736~6621320,72347956~6121902,101146541~1069076,114140049~10701635,131400499~8153873,213909504~4528444,260046848~8453657,289406976~8582532,308958519~7144889,331350016~7144515,345377430~7491289,385875968~4001241] 2024-03-07T03:49:18.681 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 7 diff [1668238~6417205,16319008~5421244,38735630~7654854,54508011~6897241,72347956~6121902,101146541~1069076,114140049~10701635,131400499~12002268,172622639~1160618,217208288~1229660,219907509~4186570,260950876~7549629,286966749~11022759,308958519~7144889,331579789~6914742,338641176~1731708,345377430~7491289,365771831~4427582,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.681 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,38735630~7654854,54508011~6897241,72347956~6121902,101146541~1069076,114140049~10701635,131400499~12002268,172622639~1160618,217208288~1229660,219907509~4186570,260950876~7549629,286966749~11022759,308958519~7144889,331579789~6914742,338641176~1731708,345377430~7491289,365771831~4427582,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.687 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,37748736~8641748,54508011~6897241,72347956~6121902,101146541~1069076,114140049~10701635,131400499~12002268,171966464~1816793,213909504~10184575,260046848~8453657,285212672~12776836,308958519~7144889,331350016~7144515,338641176~1731708,345377430~7491289,365771831~4427582,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.687 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 8 diff [1668238~6417205,16319008~5421244,38735630~7654854,54508011~6897241,71880073~7248390,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,172622639~1160618,217208288~1229660,219907509~4186570,235744159~5943951,260950876~7549629,283198497~14791011,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.687 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,38735630~7654854,54508011~6897241,71880073~7248390,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,172622639~1160618,217208288~1229660,219907509~4186570,235744159~5943951,260950876~7549629,283198497~14791011,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.694 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,37748736~8641748,54508011~6897241,71880073~7248390,101146541~1069076,106662570~2349710,114140049~10701635,131400499~12002268,171966464~1816793,213909504~10184575,235744159~5943951,260046848~8453657,281018368~16971140,308958519~9830820,331350016~7144515,338641176~1731708,345377430~7491289,365771831~9748242,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.694 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 9 diff [1668238~6417205,16319008~5421244,38735630~7654854,54508011~6897241,68844351~10284112,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,143458483~109788,172622639~1160618,217208288~1229660,219907509~8387335,235744159~5943951,258627946~10115087,271980343~4505078,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.694 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,38735630~7654854,54508011~6897241,68844351~10284112,101146541~1069076,107640648~1371632,114140049~10701635,131400499~12002268,143458483~109788,172622639~1160618,217208288~1229660,219907509~8387335,235744159~5943951,258627946~10115087,271980343~4505078,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.701 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,37748736~8641748,54508011~6897241,68844351~10284112,101146541~1069076,106662570~2349710,114140049~10701635,131400499~12002268,143458483~109788,171966464~1816793,213909504~14385340,235744159~5943951,255852544~12890489,271980343~4505078,276824064~21165444,308958519~9830820,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.701 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 10 diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,54508011~6897241,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~8387335,235744159~5943951,248347320~4168563,258627946~10115087,271980343~4809701,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.701 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,54508011~6897241,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~8387335,235744159~5943951,248347320~4168563,258627946~10115087,271980343~4809701,278150166~19839342,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.708 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,27243122~8307066,37748736~8641748,54508011~6897241,68844351~16171387,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,171966464~6682001,187362711~1171839,213909504~14385340,235744159~5943951,248347320~4168563,255852544~12890489,271980343~4809701,276824064~21165444,308958519~9830820,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.708 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 11 diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12085719,235744159~5943951,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.708 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~5421244,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12085719,235744159~5943951,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.715 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~9157340,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,154546637~1285898,171966464~6682001,187362711~1171839,213909504~18083724,235744159~5943951,248347320~4790785,255852544~12890489,271980343~4809701,276824064~21165444,308958519~9830820,322402909~5512743,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.715 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 12 diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.715 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.721 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,154546637~1285898,171966464~6682001,187362711~1171839,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~25748982,308958519~9830820,322402909~5512743,331350016~7144515,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.721 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 13 diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.722 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.728 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~3278595,187362711~1171839,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~25748982,308958519~9830820,322402909~5512743,327937615~10556916,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.728 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 14 diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.728 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.735 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~3278595,187362711~1171839,205520896~7904290,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~25748982,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.735 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 15 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.735 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.743 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~25748982,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.743 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 16 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10488026,271980343~4809701,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.743 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10488026,271980343~4809701,278150166~19839342,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.750 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~8641748,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,255852544~13263428,271980343~4809701,276824064~25748982,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.750 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 17 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.750 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,107640648~1371632,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~3874708,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.756 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,106662570~2349710,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~936151,235744159~5943951,243876757~4426093,248347320~20768652,271980343~4809701,276824064~27353295,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~6303439] 2024-03-07T03:49:18.757 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 18 diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.757 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1668238~6417205,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.764 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1668238~6417205,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.764 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 19 diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.764 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.770 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.771 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 6 diff [3758582~4326861,16319008~5418293,38735630~5634426,217208288~1229660,260950876~7549629,308958519~7144889,333531803~4962728,345377430~7491289,388314730~1562479] 2024-03-07T03:49:18.771 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~5418293,38735630~5634426,217208288~1229660,260950876~7549629,308958519~7144889,333531803~4962728,345377430~7491289,388314730~1562479] 2024-03-07T03:49:18.776 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~9154389,37748736~6621320,213909504~4528444,260046848~8453657,308958519~7144889,333531803~4962728,345377430~7491289,385875968~4001241] 2024-03-07T03:49:18.776 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 7 diff [3758582~4326861,16319008~5421244,38735630~7654854,54508011~6897241,136109713~7293054,172622639~1160618,217208288~1229660,219907509~4186570,260950876~7549629,286966749~7178032,308958519~7144889,333531803~4962728,338641176~1731708,345377430~7491289,365771831~4427582,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.777 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~5421244,38735630~7654854,54508011~6897241,136109713~7293054,172622639~1160618,217208288~1229660,219907509~4186570,260950876~7549629,286966749~7178032,308958519~7144889,333531803~4962728,338641176~1731708,345377430~7491289,365771831~4427582,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.783 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~9157340,37748736~8641748,54508011~6897241,136109713~7293054,171966464~1816793,213909504~10184575,260046848~8453657,285212672~8932109,308958519~7144889,333531803~4962728,338641176~1731708,345377430~7491289,365771831~4427582,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.783 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 8 diff [3758582~4326861,16319008~5421244,38735630~7654854,54508011~6897241,71880073~7248390,107640648~1371632,135426417~7976350,172622639~1160618,217208288~1229660,219907509~4186570,235744159~5943951,260950876~7549629,283198497~10946284,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.783 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~5421244,38735630~7654854,54508011~6897241,71880073~7248390,107640648~1371632,135426417~7976350,172622639~1160618,217208288~1229660,219907509~4186570,235744159~5943951,260950876~7549629,283198497~10946284,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.790 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~9157340,37748736~8641748,54508011~6897241,71880073~7248390,106662570~2349710,135426417~7976350,171966464~1816793,213909504~10184575,235744159~5943951,260046848~8453657,281018368~13126413,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,365771831~9748242,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.790 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 9 diff [3758582~4326861,16319008~5421244,38735630~7654854,54508011~6897241,68844351~10284112,107640648~1371632,135426417~7976350,143458483~109788,172622639~1160618,217208288~1229660,219907509~8387335,235744159~5943951,258627946~10115087,271980343~4505078,278150166~15994615,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.790 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~5421244,38735630~7654854,54508011~6897241,68844351~10284112,107640648~1371632,135426417~7976350,143458483~109788,172622639~1160618,217208288~1229660,219907509~8387335,235744159~5943951,258627946~10115087,271980343~4505078,278150166~15994615,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.796 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~9157340,37748736~8641748,54508011~6897241,68844351~10284112,106662570~2349710,135426417~7976350,143458483~109788,171966464~1816793,213909504~14385340,235744159~5943951,255852544~12890489,271980343~4505078,276824064~17320717,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.796 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 10 diff [3758582~4326861,16319008~5421244,27243122~8307066,38735630~7654854,54508011~6897241,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~8387335,235744159~5943951,248347320~4168563,258627946~10115087,271980343~4809701,278150166~15994615,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.796 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~5421244,27243122~8307066,38735630~7654854,54508011~6897241,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~8387335,235744159~5943951,248347320~4168563,258627946~10115087,271980343~4809701,278150166~15994615,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.802 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~9157340,27243122~8307066,37748736~8641748,54508011~6897241,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,171966464~6682001,187362711~1171839,213909504~14385340,235744159~5943951,248347320~4168563,255852544~12890489,271980343~4809701,276824064~17320717,308958519~9830820,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,365771831~9748242,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.803 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 11 diff [3758582~4326861,16319008~5421244,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12085719,235744159~5943951,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.803 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~5421244,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12085719,235744159~5943951,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,388314730~1562479,394784133~2473510] 2024-03-07T03:49:18.809 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~9157340,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,171966464~6682001,187362711~1171839,213909504~18083724,235744159~5943951,248347320~4790785,255852544~12890489,271980343~4809701,276824064~20136910,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,385875968~4001241,394264576~2993067] 2024-03-07T03:49:18.809 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 12 diff [3758582~4326861,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.809 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.816 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~12098108,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,171966464~6682001,187362711~1171839,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~20136910,297989508~4583538,308958519~9830820,322402909~5512743,331500133~6994398,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.816 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 13 diff [3758582~4326861,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.816 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~9830820,322402909~5512743,330544483~7950048,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.824 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~12098108,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~3278595,187362711~1171839,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~20136910,297989508~4583538,308958519~9830820,322402909~5512743,327937615~10556916,338641176~1731708,345377430~7491289,357034236~4493146,362085616~13434457,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.824 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 14 diff [3758582~4326861,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.825 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,16319008~8362012,27243122~8307066,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.831 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~12098108,27243122~8307066,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,93923603~1285999,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~3278595,187362711~1171839,205520896~7904290,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~20136910,297989508~4583538,308958519~29536012,338641176~1731708,345377430~7491289,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.831 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 15 diff [3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.831 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10115087,271980343~4809701,278150166~18810808,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.838 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3758582~4326861,12582912~12098108,27243122~10370492,37748736~8641748,54508011~8219020,62914560~5338260,68844351~16171387,93923603~1285999,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,235744159~5943951,243876757~4426093,248347320~4790785,255852544~12890489,271980343~4809701,276824064~20136910,297989508~4583538,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.838 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 16 diff [3099290~416434,3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10488026,271980343~4809701,278150166~18810808,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.838 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7654854,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~10488026,271980343~4809701,278150166~18810808,300302651~2270395,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.846 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,3758582~4326861,12582912~12098108,27243122~10370492,37748736~8641748,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,255852544~13263428,271980343~4809701,276824064~20136910,297989508~4583538,308958519~31414365,345377430~7491289,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824] 2024-03-07T03:49:18.846 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 17 diff [3099290~416434,3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~3874708,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.846 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~3874708,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.853 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,3758582~4326861,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~936151,235744159~5943951,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~6187851,308958519~31414365,345377430~8026319,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~6303439] 2024-03-07T03:49:18.853 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 18 diff [3099290~416434,3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.853 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,3758582~4326861,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.860 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,3758582~4326861,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.860 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 19 diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.860 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.867 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.867 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 7 diff [20871198~869054,43299722~3090762,54508011~6897241,136109713~7293054,172622639~1160618,219907509~4186570,286966749~7178032,338641176~1731708,365771831~4427582,394784133~2473510] 2024-03-07T03:49:18.868 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20871198~869054,43299722~3090762,54508011~6897241,136109713~7293054,172622639~1160618,219907509~4186570,286966749~7178032,338641176~1731708,365771831~4427582,394784133~2473510] 2024-03-07T03:49:18.876 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20871198~869054,43299722~3090762,54508011~6897241,136109713~7293054,171966464~1816793,218437948~5656131,285212672~8932109,338641176~1731708,365771831~4427582,394264576~2993067] 2024-03-07T03:49:18.876 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 8 diff [20871198~869054,43299722~3090762,54508011~6897241,71880073~7248390,107640648~1371632,135426417~7976350,172622639~1160618,219907509~4186570,235744159~5943951,283198497~10946284,311846963~6942376,331500133~3539963,338641176~1731708,365771831~9748242,394784133~2473510] 2024-03-07T03:49:18.876 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20871198~869054,43299722~3090762,54508011~6897241,71880073~7248390,107640648~1371632,135426417~7976350,172622639~1160618,219907509~4186570,235744159~5943951,283198497~10946284,311846963~6942376,331500133~3539963,338641176~1731708,365771831~9748242,394784133~2473510] 2024-03-07T03:49:18.881 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20871198~869054,43299722~3090762,54508011~6897241,71880073~7248390,106662570~2349710,135426417~7976350,171966464~1816793,218437948~5656131,235744159~5943951,281018368~13126413,311846963~6942376,331500133~3539963,338641176~1731708,365771831~9748242,394264576~2993067] 2024-03-07T03:49:18.881 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 9 diff [20871198~869054,43299722~3090762,54508011~6897241,68844351~10284112,107640648~1371632,135426417~7976350,143458483~109788,172622639~1160618,219907509~8387335,235744159~5943951,258627946~3865095,268107668~635365,271980343~4505078,278150166~15994615,311846963~6942376,331500133~3539963,338641176~1731708,357034236~4493146,365771831~9748242,394784133~2473510] 2024-03-07T03:49:18.881 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20871198~869054,43299722~3090762,54508011~6897241,68844351~10284112,107640648~1371632,135426417~7976350,143458483~109788,172622639~1160618,219907509~8387335,235744159~5943951,258627946~3865095,268107668~635365,271980343~4505078,278150166~15994615,311846963~6942376,331500133~3539963,338641176~1731708,357034236~4493146,365771831~9748242,394784133~2473510] 2024-03-07T03:49:18.888 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20871198~869054,43299722~3090762,54508011~6897241,68844351~10284112,106662570~2349710,135426417~7976350,143458483~109788,171966464~1816793,218437948~9856896,235744159~5943951,255852544~6640497,268107668~635365,271980343~4505078,276824064~17320717,311846963~6942376,331500133~3539963,338641176~1731708,357034236~4493146,365771831~9748242,394264576~2993067] 2024-03-07T03:49:18.888 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 10 diff [20871198~869054,27243122~8307066,42537032~3853452,54508011~6897241,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,172622639~1160618,175143458~3505007,187362711~1171839,219907509~8387335,235744159~5943951,248347320~4168563,258627946~3865095,268107668~635365,271980343~4809701,278150166~15994615,311846963~6942376,331500133~3539963,338641176~1731708,357034236~4493146,365771831~9748242,394784133~2473510] 2024-03-07T03:49:18.888 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20871198~869054,27243122~8307066,42537032~3853452,54508011~6897241,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,172622639~1160618,175143458~3505007,187362711~1171839,219907509~8387335,235744159~5943951,248347320~4168563,258627946~3865095,268107668~635365,271980343~4809701,278150166~15994615,311846963~6942376,331500133~3539963,338641176~1731708,357034236~4493146,365771831~9748242,394784133~2473510] 2024-03-07T03:49:18.895 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20871198~869054,27243122~8307066,42537032~3853452,54508011~6897241,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,171966464~6682001,187362711~1171839,218437948~9856896,235744159~5943951,248347320~4168563,255852544~6640497,268107668~635365,271980343~4809701,276824064~17320717,311846963~6942376,331500133~3539963,338641176~1731708,357034236~4493146,365771831~9748242,394264576~2993067] 2024-03-07T03:49:18.895 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 11 diff [20871198~869054,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,219907509~12085719,235744159~5943951,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,311846963~6942376,322402909~5512743,331500133~3539963,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,394784133~2473510] 2024-03-07T03:49:18.895 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20871198~869054,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,219907509~12085719,235744159~5943951,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,311846963~6942376,322402909~5512743,331500133~3539963,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,394784133~2473510] 2024-03-07T03:49:18.901 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20871198~869054,27243122~8307066,42537032~3853452,54508011~8219020,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,171966464~6682001,187362711~1171839,218437948~13555280,235744159~5943951,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~20136910,311846963~6942376,322402909~5512743,331500133~3539963,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,394264576~2993067] 2024-03-07T03:49:18.901 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 12 diff [20376114~4304906,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,331500133~3539963,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.902 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20376114~4304906,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,172622639~1160618,175143458~3505007,187362711~1171839,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,331500133~3539963,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394784133~6342267] 2024-03-07T03:49:18.908 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20376114~4304906,27243122~8307066,42537032~3853452,54508011~8219020,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,154546637~1285898,171966464~6682001,187362711~1171839,218437948~14447916,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~6942376,322402909~5512743,331500133~3539963,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394264576~6861824] 2024-03-07T03:49:18.908 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 13 diff [18888173~5792847,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,330544483~7413135,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.908 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,330544483~7413135,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.914 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~3853452,54508011~8219020,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~3278595,187362711~1171839,218437948~14447916,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~6942376,322402909~5512743,327937615~10020003,338641176~1731708,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394264576~6861824] 2024-03-07T03:49:18.915 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 14 diff [18888173~5792847,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~26110655,338641176~1731708,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.915 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~26110655,338641176~1731708,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.922 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~3853452,54508011~8219020,62914560~5338260,68844351~16171387,93923603~1285999,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~3278595,187362711~1171839,205520896~7904290,218437948~14447916,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,263519614~632790,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~26110655,338641176~1731708,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824] 2024-03-07T03:49:18.922 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 15 diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28525921,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.922 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~3853452,54508011~8219020,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28525921,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.929 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~9284123,27243122~10370492,42537032~3853452,54508011~8219020,62914560~5338260,68844351~16171387,93923603~1285999,106662570~2349710,112841617~4261226,135426417~7976350,143458483~109788,147091666~5234760,154546637~8150013,171966464~6682001,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~28525921,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824] 2024-03-07T03:49:18.929 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 16 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~3853452,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263372186~5743786,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28525921,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.929 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~3853452,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263372186~5743786,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28525921,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082] 2024-03-07T03:49:18.935 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,42537032~3853452,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,263372186~5743786,271980343~4809701,276824064~20136910,297989508~4583538,311846963~28525921,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824] 2024-03-07T03:49:18.936 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 17 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~3874708,311846963~28525921,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.936 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~3874708,311846963~28525921,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,410052091~3098836] 2024-03-07T03:49:18.942 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~936151,235744159~5943951,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~6187851,311846963~28525921,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~6303439] 2024-03-07T03:49:18.942 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 18 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.942 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.949 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.949 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 19 diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.950 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:18.956 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:18.956 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 8 diff [57610260~405200,71880073~7248390,107640648~1371632,135426417~7906202,235744159~5943951,283198497~9374773,311846963~6942376,331500133~3539963,369938543~5581530] 2024-03-07T03:49:18.956 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [57610260~405200,71880073~7248390,107640648~1371632,135426417~7906202,235744159~5943951,283198497~9374773,311846963~6942376,331500133~3539963,369938543~5581530] 2024-03-07T03:49:18.963 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [57610260~405200,71880073~7248390,106662570~2349710,135426417~7906202,235744159~5943951,281018368~11554902,311846963~6942376,331500133~3539963,369938543~5581530] 2024-03-07T03:49:18.963 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 9 diff [57610260~405200,68844351~10284112,107640648~1371632,135426417~7906202,143458483~109788,220902876~7391968,235744159~5943951,258627946~3865095,268107668~635365,271980343~4505078,278150166~14423104,311846963~6942376,331500133~3539963,357034236~4493146,369938543~5581530] 2024-03-07T03:49:18.963 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [57610260~405200,68844351~10284112,107640648~1371632,135426417~7906202,143458483~109788,220902876~7391968,235744159~5943951,258627946~3865095,268107668~635365,271980343~4505078,278150166~14423104,311846963~6942376,331500133~3539963,357034236~4493146,369938543~5581530] 2024-03-07T03:49:18.970 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [57610260~405200,68844351~10284112,106662570~2349710,135426417~7906202,143458483~109788,220902876~7391968,235744159~5943951,255852544~6640497,268107668~635365,271980343~4505078,276824064~15749206,311846963~6942376,331500133~3539963,357034236~4493146,369938543~5581530] 2024-03-07T03:49:18.970 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 10 diff [27243122~8307066,42537032~1697190,57610260~405200,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,175143458~3505007,187362711~1171839,220902876~7391968,235744159~5943951,248347320~4168563,258627946~3865095,268107668~635365,271980343~4809701,278150166~14423104,311846963~6942376,331500133~3539963,357034236~4493146,369938543~5581530] 2024-03-07T03:49:18.970 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [27243122~8307066,42537032~1697190,57610260~405200,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,175143458~3505007,187362711~1171839,220902876~7391968,235744159~5943951,248347320~4168563,258627946~3865095,268107668~635365,271980343~4809701,278150166~14423104,311846963~6942376,331500133~3539963,357034236~4493146,369938543~5581530] 2024-03-07T03:49:18.977 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [27243122~8307066,42537032~1697190,57610260~405200,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7906202,143458483~109788,173783257~4865208,187362711~1171839,220902876~7391968,235744159~5943951,248347320~4168563,255852544~6640497,268107668~635365,271980343~4809701,276824064~15749206,311846963~6942376,331500133~3539963,357034236~4493146,369938543~5581530] 2024-03-07T03:49:18.977 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 11 diff [27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11090352,235744159~5943951,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~14423104,292812639~4148335,311846963~6942376,322402909~5512743,331500133~3539963,357034236~4493146,362085616~6636924,369938543~5581530,375723852~2962468] 2024-03-07T03:49:18.978 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11090352,235744159~5943951,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~14423104,292812639~4148335,311846963~6942376,322402909~5512743,331500133~3539963,357034236~4493146,362085616~6636924,369938543~5581530,375723852~2962468] 2024-03-07T03:49:18.984 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7906202,143458483~109788,154546637~1285898,173783257~4865208,187362711~1171839,220902876~11090352,235744159~5943951,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~15749206,292812639~4148335,311846963~6942376,322402909~5512743,331500133~3539963,357034236~4493146,362085616~6636924,369938543~5581530,375723852~2962468] 2024-03-07T03:49:18.984 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 12 diff [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,331500133~3539963,357034236~4493146,362085616~6636924,369938543~5581530,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:18.985 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,331500133~3539963,357034236~4493146,362085616~6636924,369938543~5581530,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:18.991 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7906202,143458483~109788,154546637~1285898,173783257~4865208,187362711~1171839,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~6942376,322402909~5512743,331500133~3539963,357034236~4493146,362085616~6636924,369938543~5581530,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:18.991 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 13 diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,330544483~7413135,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:18.991 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~6942376,322402909~5512743,330544483~7413135,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:18.998 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~16171387,106662570~2349710,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~8150013,173783257~4865208,180903040~3278595,187362711~1171839,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~6942376,322402909~5512743,327937615~10020003,357034236~4493146,362085616~13434457,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:18.998 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 14 diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~26110655,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:18.998 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~26110655,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.004 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~16171387,93923603~1285999,106662570~2349710,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~8150013,173783257~4865208,180903040~3278595,187362711~1171839,205520896~7904290,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,263519614~632790,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~26110655,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.005 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 15 diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28390243,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.005 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,107640648~1371632,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28390243,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.011 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~9284123,27243122~10370492,42537032~1697190,56594107~6132924,62914560~5338260,68844351~16171387,93923603~1285999,106662570~2349710,112841617~4261226,135426417~7906202,143458483~109788,147091666~5234760,154546637~8150013,173783257~4865208,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,276824064~20136910,297989508~4583538,311846963~28390243,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.011 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 16 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263372186~5743786,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28390243,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.011 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,258627946~3865095,263372186~5743786,271980343~4809701,278150166~18810808,300302651~2270395,311846963~28390243,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.017 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,42537032~1697190,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~936151,235744159~5943951,243876757~4426093,248347320~4790785,255852544~6640497,263372186~5743786,271980343~4809701,276824064~20136910,297989508~4583538,311846963~28390243,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.017 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 17 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~3874708,311846963~28390243,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.018 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,235744159~5943951,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~3874708,311846963~28390243,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.024 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~936151,235744159~5943951,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~6187851,311846963~28390243,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~6303439] 2024-03-07T03:49:19.024 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 18 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.024 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.031 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,349211446~4192303,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.031 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 19 diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.031 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.037 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.037 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 9 diff [68844351~5011596,143458483~109788,220902876~7391968,258627946~3865095,268107668~635365,271980343~4505078,278150166~7655540,357034236~4493146] 2024-03-07T03:49:19.038 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [68844351~5011596,143458483~109788,220902876~7391968,258627946~3865095,268107668~635365,271980343~4505078,278150166~7655540,357034236~4493146] 2024-03-07T03:49:19.044 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [68844351~5011596,143458483~109788,220902876~7391968,255852544~6640497,268107668~635365,271980343~4505078,276824064~8981642,357034236~4493146] 2024-03-07T03:49:19.044 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 10 diff [27243122~8307066,42537032~1697190,68844351~5011596,78559315~6456423,112841617~4261226,143458483~109788,175143458~3505007,187362711~1171839,220902876~7391968,248347320~4168563,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,357034236~4493146] 2024-03-07T03:49:19.044 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [27243122~8307066,42537032~1697190,68844351~5011596,78559315~6456423,112841617~4261226,143458483~109788,175143458~3505007,187362711~1171839,220902876~7391968,248347320~4168563,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,357034236~4493146] 2024-03-07T03:49:19.051 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [27243122~8307066,42537032~1697190,68844351~5011596,78559315~6456423,112841617~4261226,143458483~109788,173783257~4865208,187362711~1171839,220902876~7391968,248347320~4168563,255852544~6640497,268107668~635365,271980343~4809701,276824064~8981642,357034236~4493146] 2024-03-07T03:49:19.051 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 11 diff [27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~5011596,78559315~6456423,112841617~4261226,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11090352,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,292812639~4148335,322402909~5512743,357034236~4493146,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.051 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~5011596,78559315~6456423,112841617~4261226,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11090352,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,292812639~4148335,322402909~5512743,357034236~4493146,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.058 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~5011596,78559315~6456423,112841617~4261226,143458483~109788,154546637~1285898,173783257~4865208,187362711~1171839,220902876~11090352,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~8981642,292812639~4148335,322402909~5512743,357034236~4493146,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.058 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 12 diff [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~5011596,75283715~9732023,112841617~4261226,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,322402909~5512743,357034236~4493146,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.058 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~5011596,75283715~9732023,112841617~4261226,143458483~109788,154546637~1285898,175143458~3505007,187362711~1171839,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,322402909~5512743,357034236~4493146,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.066 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~5011596,75283715~9732023,112841617~4261226,143458483~109788,154546637~1285898,173783257~4865208,187362711~1171839,220902876~11982988,243876757~4426093,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~8981642,287521464~9439510,297989508~4583538,322402909~5512743,357034236~4493146,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.066 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 13 diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~5011596,75283715~9732023,112841617~4261226,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,322402909~5512743,330544483~7413135,357034236~4493146,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.066 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~5011596,75283715~9732023,112841617~4261226,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,322402909~5512743,330544483~7413135,357034236~4493146,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.073 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~5011596,75283715~9732023,112841617~4261226,143458483~109788,147091666~5234760,154546637~8150013,173783257~4865208,180903040~3278595,187362711~1171839,220902876~11982988,243876757~4426093,248347320~4790785,255852544~6640497,268107668~635365,271980343~4809701,276824064~8981642,287521464~9439510,297989508~4583538,322402909~5512743,327937615~10020003,357034236~4493146,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.073 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 14 diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,112841617~4261226,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,315737520~22220098,357034236~13401323,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.073 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,112841617~4261226,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,315737520~22220098,357034236~13401323,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.079 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,68844351~16171387,93923603~1285999,112841617~4261226,143458483~109788,147091666~5234760,154546637~8150013,173783257~4865208,180903040~3278595,187362711~1171839,205520896~7904290,220902876~11982988,243876757~4426093,248347320~4790785,255852544~6640497,263519614~632790,268107668~635365,271980343~4809701,276824064~8981642,287521464~9439510,297989508~4583538,315737520~22220098,357034236~13401323,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.079 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 15 diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,112841617~4261226,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,315737520~24499686,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.079 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~16171387,93923603~1285999,112841617~4261226,143458483~109788,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,243876757~4426093,248347320~4790785,258627946~3865095,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,315737520~24499686,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.086 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~9284123,27243122~10370492,42537032~1697190,56594107~6132924,62914560~5338260,68844351~16171387,93923603~1285999,112841617~4261226,143458483~109788,147091666~5234760,154546637~8150013,173783257~4865208,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,243876757~4426093,248347320~4790785,255852544~6640497,263519614~632790,265268048~2824484,268107668~635365,271980343~4809701,276824064~8981642,287521464~9439510,297989508~4583538,315737520~24499686,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.086 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 16 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,112841617~4261226,118268023~5048976,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,243876757~4426093,248347320~4790785,258627946~3865095,263372186~5743786,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,312404032~1018043,315737520~24499686,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.086 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,112841617~4261226,118268023~5048976,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,243876757~4426093,248347320~4790785,258627946~3865095,263372186~5743786,271980343~4809701,278150166~7655540,287521464~9439510,300302651~2270395,312404032~1018043,315737520~24499686,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.092 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,42537032~1697190,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,112841617~4261226,118268023~5048976,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~936151,243876757~4426093,248347320~4790785,255852544~6640497,263372186~5743786,271980343~4809701,276824064~8981642,287521464~9439510,297989508~4583538,312404032~1018043,315737520~24499686,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.092 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 17 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,112841617~4261226,118268023~5048976,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~8102843,287521464~9439510,300302651~3874708,312404032~1018043,315737520~24499686,349211446~4192303,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.092 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,112841617~4261226,118268023~5048976,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~936151,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~8102843,287521464~9439510,300302651~3874708,312404032~1018043,315737520~24499686,349211446~4192303,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.098 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,112841617~4261226,118268023~5048976,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~936151,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~8981642,287521464~9439510,297989508~6187851,312404032~1018043,315737520~24499686,349211446~4192303,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~6303439] 2024-03-07T03:49:19.098 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 18 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,349211446~4192303,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.098 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,349211446~4192303,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.104 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,349211446~4192303,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.104 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 19 diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.104 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.110 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.110 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 10 diff [27243122~8307066,42537032~1697190,78559315~6456423,112841617~4261226,175143458~3505007,187362711~1171839,248347320~4168563,274300269~2489775] 2024-03-07T03:49:19.111 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [27243122~8307066,42537032~1697190,78559315~6456423,112841617~4261226,175143458~3505007,187362711~1171839,248347320~4168563,274300269~2489775] 2024-03-07T03:49:19.117 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [27243122~8307066,42537032~1697190,78559315~6456423,112841617~4261226,173783257~4865208,187362711~1171839,248347320~4168563,274300269~2489775] 2024-03-07T03:49:19.117 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 11 diff [27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,78559315~6456423,112841617~4261226,154546637~1285898,175143458~3505007,187362711~1171839,224740997~7252231,248347320~4790785,274300269~2489775,292812639~4148335,322402909~5512743,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.117 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,78559315~6456423,112841617~4261226,154546637~1285898,175143458~3505007,187362711~1171839,224740997~7252231,248347320~4790785,274300269~2489775,292812639~4148335,322402909~5512743,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.123 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,78559315~6456423,112841617~4261226,154546637~1285898,173783257~4865208,187362711~1171839,224740997~7252231,248347320~4790785,274300269~2489775,292812639~4148335,322402909~5512743,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.123 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 12 diff [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,75283715~9732023,112841617~4261226,154546637~1285898,175143458~3505007,187362711~1171839,224740997~8144867,243876757~4426093,248347320~4790785,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.123 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,75283715~9732023,112841617~4261226,154546637~1285898,175143458~3505007,187362711~1171839,224740997~8144867,243876757~4426093,248347320~4790785,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.129 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20376114~4304906,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,75283715~9732023,112841617~4261226,154546637~1285898,173783257~4865208,187362711~1171839,224740997~8144867,243876757~4426093,248347320~4790785,274300269~2489775,282524789~1540418,287521464~9439510,297989508~4583538,322402909~5512743,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.129 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 13 diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,73270615~516924,75283715~9732023,112841617~4261226,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,224740997~8144867,243876757~4426093,248347320~4790785,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,330544483~7413135,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.129 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,73270615~516924,75283715~9732023,112841617~4261226,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,224740997~8144867,243876757~4426093,248347320~4790785,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,330544483~7413135,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.135 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,73270615~516924,75283715~9732023,112841617~4261226,147091666~5234760,154546637~8150013,173783257~4865208,180903040~3278595,187362711~1171839,224740997~8144867,243876757~4426093,248347320~4790785,274300269~2489775,282524789~1540418,287521464~9439510,297989508~4583538,322402909~5512743,327937615~10020003,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.135 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 14 diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,69963884~15051854,93923603~1285999,112841617~4261226,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,224740997~8144867,243876757~4426093,248347320~4790785,263519614~632790,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,315737520~22220098,358137084~12298475,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.135 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,66725534~1527286,69963884~15051854,93923603~1285999,112841617~4261226,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~3278595,187362711~1171839,207908122~5517064,224740997~8144867,243876757~4426093,248347320~4790785,263519614~632790,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,315737520~22220098,358137084~12298475,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.141 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,27243122~8307066,42537032~1697190,56594107~6132924,62914560~5338260,69963884~15051854,93923603~1285999,112841617~4261226,147091666~5234760,154546637~8150013,173783257~4865208,180903040~3278595,187362711~1171839,205520896~7904290,224740997~8144867,243876757~4426093,248347320~4790785,263519614~632790,274300269~2489775,282524789~1540418,287521464~9439510,297989508~4583538,315737520~22220098,358137084~12298475,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.141 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 15 diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,69963884~15051854,93923603~1285999,112841617~4261226,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,243876757~4426093,248347320~4790785,263519614~632790,265268048~2824484,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.141 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,69963884~15051854,93923603~1285999,112841617~4261226,147091666~5234760,154546637~1285898,158985649~3711001,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,243876757~4426093,248347320~4790785,263519614~632790,265268048~2824484,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.147 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~9284123,27243122~10370492,42537032~1697190,56594107~6132924,62914560~5338260,69963884~15051854,93923603~1285999,112841617~4261226,147091666~5234760,154546637~8150013,173783257~4865208,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,243876757~4426093,248347320~4790785,263519614~632790,265268048~2824484,274300269~2489775,282524789~1540418,287521464~9439510,297989508~4583538,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.147 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 16 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112841617~4261226,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,248347320~4790785,263372186~5743786,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,312404032~1018043,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.147 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,42537032~1697190,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112841617~4261226,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,248347320~4790785,263372186~5743786,274300269~2489775,282524789~1540418,287521464~9439510,300302651~2270395,312404032~1018043,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.153 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,42537032~1697190,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,112841617~4261226,118268023~5048976,147091666~5234760,154546637~14125440,173783257~4865208,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~936151,243876757~4426093,248347320~4790785,263372186~5743786,274300269~2489775,282524789~1540418,287521464~9439510,297989508~4583538,312404032~1018043,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.153 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 17 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112841617~4261226,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,248347320~9156972,263372186~5743786,274300269~2489775,277702863~6362344,287521464~9439510,300302651~3874708,312404032~1018043,315737520~24499686,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.154 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112841617~4261226,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,248347320~9156972,263372186~5743786,274300269~2489775,277702863~6362344,287521464~9439510,300302651~3874708,312404032~1018043,315737520~24499686,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.161 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,112841617~4261226,118268023~5048976,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~936151,243876757~4426093,248347320~9156972,263372186~5743786,274300269~2489775,277702863~6362344,287521464~9439510,297989508~6187851,312404032~1018043,315737520~24499686,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~6303439] 2024-03-07T03:49:19.161 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 18 diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,274300269~2489775,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.161 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,274300269~2489775,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.166 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,274300269~2489775,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.166 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 19 diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.166 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.173 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.173 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 11 diff [56594107~6132924,66725534~1527286,154546637~1285898,224740997~7252231,249454878~3683227,292812639~4148335,322402909~5512743,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.173 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [56594107~6132924,66725534~1527286,154546637~1285898,224740997~7252231,249454878~3683227,292812639~4148335,322402909~5512743,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.179 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [56594107~6132924,62914560~5338260,154546637~1285898,224740997~7252231,249454878~3683227,292812639~4148335,322402909~5512743,362085616~6636924,375723852~2962468] 2024-03-07T03:49:19.179 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 12 diff [20376114~4304906,31904019~2211031,56594107~6132924,66725534~1527286,75283715~3751471,154546637~1285898,224740997~8144867,243876757~4426093,249454878~3683227,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.179 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20376114~4304906,31904019~2211031,56594107~6132924,66725534~1527286,75283715~3751471,154546637~1285898,224740997~8144867,243876757~4426093,249454878~3683227,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.184 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20376114~4304906,31904019~2211031,56594107~6132924,62914560~5338260,75283715~3751471,154546637~1285898,224740997~8144867,243876757~4426093,249454878~3683227,282524789~1540418,287521464~9439510,297989508~4583538,322402909~5512743,362085616~6636924,375723852~2962468,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.184 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 13 diff [18888173~5792847,31904019~2211031,56594107~6132924,66725534~1527286,73270615~516924,75283715~3751471,147091666~5234760,154546637~1285898,158985649~3711001,180903040~3278595,224740997~8144867,243876757~4426093,249454878~3683227,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,330544483~7413135,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.184 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,31904019~2211031,56594107~6132924,66725534~1527286,73270615~516924,75283715~3751471,147091666~5234760,154546637~1285898,158985649~3711001,180903040~3278595,224740997~8144867,243876757~4426093,249454878~3683227,282524789~1540418,287521464~9439510,300302651~2270395,322402909~5512743,330544483~7413135,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.191 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,31904019~2211031,56594107~6132924,62914560~5338260,73270615~516924,75283715~3751471,147091666~5234760,154546637~8150013,180903040~3278595,224740997~8144867,243876757~4426093,249454878~3683227,282524789~1540418,287521464~9439510,297989508~4583538,322402909~5512743,327937615~10020003,362085616~8349943,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.191 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 14 diff [18888173~5792847,31904019~2211031,56594107~6132924,66725534~1527286,69963884~9071302,93923603~1285999,147091666~5234760,154546637~1285898,158985649~3711001,180903040~3278595,207908122~5517064,224740997~8144867,243876757~4426093,249454878~3683227,263519614~632790,282524789~1540418,287521464~9439510,300302651~2270395,315737520~22220098,358137084~12298475,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.191 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,31904019~2211031,56594107~6132924,66725534~1527286,69963884~9071302,93923603~1285999,147091666~5234760,154546637~1285898,158985649~3711001,180903040~3278595,207908122~5517064,224740997~8144867,243876757~4426093,249454878~3683227,263519614~632790,282524789~1540418,287521464~9439510,300302651~2270395,315737520~22220098,358137084~12298475,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.196 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,31904019~2211031,56594107~6132924,62914560~5338260,69963884~9071302,93923603~1285999,147091666~5234760,154546637~8150013,180903040~3278595,205520896~7904290,224740997~8144867,243876757~4426093,249454878~3683227,263519614~632790,282524789~1540418,287521464~9439510,297989508~4583538,315737520~22220098,358137084~12298475,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.197 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 15 diff [15396897~9284123,31904019~2211031,35879825~1733789,56594107~6132924,66725534~1527286,69963884~9071302,93923603~1285999,147091666~5234760,154546637~1285898,158985649~3711001,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,243876757~4426093,249454878~3683227,263519614~632790,265268048~2824484,282524789~1540418,287521464~9439510,300302651~2270395,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.197 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~9284123,31904019~2211031,35879825~1733789,56594107~6132924,66725534~1527286,69963884~9071302,93923603~1285999,147091666~5234760,154546637~1285898,158985649~3711001,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,243876757~4426093,249454878~3683227,263519614~632790,265268048~2824484,282524789~1540418,287521464~9439510,300302651~2270395,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.203 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~9284123,31904019~2211031,35550188~2063426,56594107~6132924,62914560~5338260,69963884~9071302,93923603~1285999,147091666~5234760,154546637~8150013,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,243876757~4426093,249454878~3683227,263519614~632790,265268048~2824484,282524789~1540418,287521464~9439510,297989508~4583538,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.203 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 16 diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,249454878~3683227,263372186~5743786,282524789~1540418,287521464~9439510,300302651~2270395,312404032~1018043,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.204 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,249454878~3683227,263372186~5743786,282524789~1540418,287521464~9439510,300302651~2270395,312404032~1018043,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.210 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,31904019~2211031,35550188~2063426,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,154546637~14125440,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~936151,243876757~4426093,249454878~3683227,263372186~5743786,282524789~1540418,287521464~9439510,297989508~4583538,312404032~1018043,315737520~24499686,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.210 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 17 diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,249454878~8049414,263372186~5743786,277702863~6362344,287521464~9439510,300302651~3874708,312404032~1018043,315737520~24499686,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.210 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~936151,243876757~4426093,249454878~8049414,263372186~5743786,277702863~6362344,287521464~9439510,300302651~3874708,312404032~1018043,315737520~24499686,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.215 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~936151,243876757~4426093,249454878~8049414,263372186~5743786,277702863~6362344,287521464~9439510,297989508~6187851,312404032~1018043,315737520~24499686,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~6303439] 2024-03-07T03:49:19.216 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 18 diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,274480103~1473964,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.216 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,274480103~1473964,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.222 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,274480103~1473964,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,349211446~4192303,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.222 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 19 diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.222 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.230 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.230 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 12 diff [20376114~4304906,31904019~2211031,75283715~3751471,226244801~6641063,243876757~4426093,282524789~1540418,287521464~6807957,300302651~2270395,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.230 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [20376114~4304906,31904019~2211031,75283715~3751471,226244801~6641063,243876757~4426093,282524789~1540418,287521464~6807957,300302651~2270395,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.236 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [20376114~4304906,31904019~2211031,75283715~3751471,226244801~6641063,243876757~4426093,282524789~1540418,287521464~6807957,297989508~4583538,386954373~6992617,396810984~4315416] 2024-03-07T03:49:19.236 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 13 diff [18888173~5792847,31904019~2211031,73270615~516924,75283715~3751471,147091666~5234760,158985649~3711001,180903040~3278595,226244801~6641063,243876757~4426093,282524789~1540418,287521464~6807957,300302651~2270395,324374225~70598,330544483~7413135,366918734~3516825,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.236 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,31904019~2211031,73270615~516924,75283715~3751471,147091666~5234760,158985649~3711001,180903040~3278595,226244801~6641063,243876757~4426093,282524789~1540418,287521464~6807957,300302651~2270395,324374225~70598,330544483~7413135,366918734~3516825,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.242 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,31904019~2211031,73270615~516924,75283715~3751471,147091666~5234760,155832535~6864115,180903040~3278595,226244801~6641063,243876757~4426093,282524789~1540418,287521464~6807957,297989508~4583538,324374225~70598,327937615~10020003,366918734~3516825,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.242 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 14 diff [18888173~5792847,31904019~2211031,69963884~9071302,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~3278595,207908122~5517064,226244801~6641063,243876757~4426093,263519614~632790,282524789~1540418,287521464~8499580,300302651~2270395,315737520~6846870,324289746~13667872,358137084~5551677,366918734~3516825,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.243 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~5792847,31904019~2211031,69963884~9071302,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~3278595,207908122~5517064,226244801~6641063,243876757~4426093,263519614~632790,282524789~1540418,287521464~8499580,300302651~2270395,315737520~6846870,324289746~13667872,358137084~5551677,366918734~3516825,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.249 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~5792847,31904019~2211031,69963884~9071302,93923603~1285999,147091666~5234760,155324570~457207,155832535~6864115,180903040~3278595,205520896~7904290,226244801~6641063,243876757~4426093,263519614~632790,282524789~1540418,287521464~8499580,297989508~4583538,315737520~6846870,324289746~13667872,358137084~5551677,366918734~3516825,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.249 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 15 diff [15396897~9284123,31904019~2211031,35879825~1733789,69963884~9071302,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,243876757~4426093,263519614~632790,265268048~2824484,282524789~1540418,287521464~8499580,300302651~2270395,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.249 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~9284123,31904019~2211031,35879825~1733789,69963884~9071302,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,243876757~4426093,263519614~632790,265268048~2824484,282524789~1540418,287521464~8499580,300302651~2270395,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.255 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~9284123,31904019~2211031,35550188~2063426,69963884~9071302,93923603~1285999,147091666~5234760,155324570~457207,155832535~6864115,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,226244801~6641063,243876757~4426093,263519614~632790,265268048~2824484,282524789~1540418,287521464~8499580,297989508~4583538,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.255 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 16 diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,233865617~936151,243876757~4426093,263372186~5743786,282524789~1540418,287521464~8499580,300302651~2270395,312404032~1018043,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.255 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,233865617~936151,243876757~4426093,263372186~5743786,282524789~1540418,287521464~8499580,300302651~2270395,312404032~1018043,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.261 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,31904019~2211031,35550188~2063426,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,155832535~12839542,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,226244801~6641063,233865617~936151,243876757~4426093,263372186~5743786,282524789~1540418,287521464~8499580,297989508~4583538,312404032~1018043,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416] 2024-03-07T03:49:19.262 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 17 diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,233865617~936151,243876757~4426093,252711974~4792318,263372186~5743786,277702863~6362344,287521464~8499580,300302651~3874708,312404032~1018043,315737520~6846870,323482505~16754701,349211446~4192303,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.262 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,233865617~936151,243876757~4426093,252711974~4792318,263372186~5743786,277702863~6362344,287521464~8499580,300302651~3874708,312404032~1018043,315737520~6846870,323482505~16754701,349211446~4192303,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,410052091~3098836] 2024-03-07T03:49:19.268 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,226244801~6641063,233865617~936151,243876757~4426093,252711974~4792318,263372186~5743786,277702863~6362344,287521464~8499580,297989508~6187851,312404032~1018043,315737520~6846870,323482505~16754701,349211446~4192303,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~6303439] 2024-03-07T03:49:19.268 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 18 diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,274480103~1473964,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,323482505~16754701,343434520~2214523,349211446~4192303,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.268 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,226244801~6641063,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,274480103~1473964,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,323482505~16754701,343434520~2214523,349211446~4192303,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.274 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,226244801~6641063,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,274480103~1473964,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,323482505~16754701,340372884~5276159,349211446~4192303,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.274 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 19 diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.274 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:19.280 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:19.280 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 13 diff [18888173~3944668,73270615~516924,147091666~5234760,158985649~3711001,180903040~3278595,324374225~70598,330544483~7413135,366918734~3516825,388807829~4661986,394504318~583478] 2024-03-07T03:49:19.280 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~3944668,73270615~516924,147091666~5234760,158985649~3711001,180903040~3278595,324374225~70598,330544483~7413135,366918734~3516825,388807829~4661986,394504318~583478] 2024-03-07T03:49:19.286 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~3944668,73270615~516924,147091666~5234760,155832535~6864115,180903040~3278595,324374225~70598,327937615~10020003,366918734~3516825,388807829~4661986,394504318~583478] 2024-03-07T03:49:19.286 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 14 diff [18888173~3944668,69963884~5451504,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~3278595,207908122~5517064,263519614~632790,290524600~5496444,315737520~6846870,324289746~13667872,358137084~5551677,366918734~3516825,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.286 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [18888173~3944668,69963884~5451504,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~3278595,207908122~5517064,263519614~632790,290524600~5496444,315737520~6846870,324289746~13667872,358137084~5551677,366918734~3516825,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.292 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [18888173~3944668,69963884~5451504,93923603~1285999,147091666~5234760,155324570~457207,155832535~6864115,180903040~3278595,205520896~7904290,263519614~632790,290524600~5496444,315737520~6846870,324289746~13667872,358137084~5551677,366918734~3516825,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.293 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 15 diff [15396897~7435944,35879825~1733789,69963884~5451504,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,263519614~632790,265268048~2824484,290524600~5496444,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.293 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~7435944,35879825~1733789,69963884~5451504,93923603~1285999,147091666~5234760,155324570~457207,158985649~3711001,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,263519614~632790,265268048~2824484,290524600~5496444,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.298 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~7435944,35550188~2063426,69963884~5451504,93923603~1285999,147091666~5234760,155324570~457207,155832535~6864115,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,263519614~632790,265268048~2824484,290524600~5496444,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.298 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 16 diff [3099290~416434,15396897~7435944,35879825~1733789,69963884~5451504,78357669~8827934,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,263372186~5743786,290524600~5496444,312404032~1018043,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.298 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~7435944,35879825~1733789,69963884~5451504,78357669~8827934,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,263372186~5743786,290524600~5496444,312404032~1018043,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.304 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~7435944,35550188~2063426,69963884~5451504,78357669~8827934,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,155832535~12839542,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,233865617~936151,263372186~5743786,290524600~5496444,312404032~1018043,315737520~6846870,323482505~16754701,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493] 2024-03-07T03:49:19.304 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 17 diff [3099290~416434,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,290524600~5496444,300469008~3708351,312404032~1018043,315737520~6846870,323482505~16754701,349211446~4192303,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,410052091~3098836] 2024-03-07T03:49:19.304 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,290524600~5496444,300469008~3708351,312404032~1018043,315737520~6846870,323482505~16754701,349211446~4192303,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,410052091~3098836] 2024-03-07T03:49:19.310 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,290524600~5496444,300469008~3708351,312404032~1018043,315737520~6846870,323482505~16754701,349211446~4192303,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~6303439] 2024-03-07T03:49:19.310 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 18 diff [3099290~416434,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,323482505~16754701,343434520~2214523,349211446~4192303,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.311 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,323482505~16754701,343434520~2214523,349211446~4192303,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.317 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,323482505~16754701,340372884~5276159,349211446~4192303,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:19.317 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 19 diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.318 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:19 smithi195 bash[66542]: cluster 2024-03-07T03:49:18.072589+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2024-03-07T03:49:19.324 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:19.324 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 14 diff [69963884~5451504,93923603~1285999,155324570~457207,207908122~5517064,263519614~632790,290524600~5496444,315737520~6846870,324289746~6442358,358137084~5551677,396876058~2804493] 2024-03-07T03:49:19.324 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [69963884~5451504,93923603~1285999,155324570~457207,207908122~5517064,263519614~632790,290524600~5496444,315737520~6846870,324289746~6442358,358137084~5551677,396876058~2804493] 2024-03-07T03:49:19.329 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [69963884~5451504,93923603~1285999,155324570~457207,205520896~7904290,263519614~632790,290524600~5496444,315737520~6846870,324289746~6442358,358137084~5551677,396876058~2804493] 2024-03-07T03:49:19.329 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 15 diff [15396897~3963058,35879825~1733789,69963884~5451504,93923603~1285999,155324570~457207,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,263519614~632790,265268048~2824484,290524600~5496444,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,358137084~5551677,367576613~5297086,396876058~2804493] 2024-03-07T03:49:19.329 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~3963058,35879825~1733789,69963884~5451504,93923603~1285999,155324570~457207,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,263519614~632790,265268048~2824484,290524600~5496444,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,358137084~5551677,367576613~5297086,396876058~2804493] 2024-03-07T03:49:19.335 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~3963058,35550188~2063426,69963884~5451504,93923603~1285999,155324570~457207,181822460~3589030,197132288~7317185,205520896~7904290,221858299~3493903,263519614~632790,265268048~2824484,290524600~5496444,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,358137084~5551677,367576613~5297086,396876058~2804493] 2024-03-07T03:49:19.336 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 16 diff [3099290~416434,15396897~3963058,35879825~1733789,69963884~5451504,78357669~8827934,93923603~1285999,118268023~5048976,155324570~457207,162665439~6006638,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,263372186~5743786,290524600~5496444,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,358137084~5551677,367576613~5297086,396876058~2804493] 2024-03-07T03:49:19.336 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~3963058,35879825~1733789,69963884~5451504,78357669~8827934,93923603~1285999,118268023~5048976,155324570~457207,162665439~6006638,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,263372186~5743786,290524600~5496444,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,358137084~5551677,367576613~5297086,396876058~2804493] 2024-03-07T03:49:19.342 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~3963058,35550188~2063426,69963884~5451504,78357669~8827934,93923603~1285999,118268023~5048976,155324570~457207,162665439~6006638,181822460~3589030,197132288~7317185,205520896~7904290,221858299~3493903,233865617~936151,263372186~5743786,290524600~5496444,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,358137084~5551677,367576613~5297086,396876058~2804493] 2024-03-07T03:49:19.342 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 17 diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,290524600~5496444,300469008~3708351,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,349211446~4192303,358137084~5551677,367576613~5297086,396876058~2804493,410052091~3098836] 2024-03-07T03:49:19.342 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,290524600~5496444,300469008~3708351,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,349211446~4192303,358137084~5551677,367576613~5297086,396876058~2804493,410052091~3098836] 2024-03-07T03:49:19.347 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~3963058,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,118268023~5048976,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,290524600~5496444,300469008~3708351,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,349211446~4192303,358137084~5551677,367576613~5297086,396876058~2804493,406847488~6303439] 2024-03-07T03:49:19.347 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 18 diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,343434520~2214523,349211446~4192303,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.348 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,343434520~2214523,349211446~4192303,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.354 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~3963058,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,112259179~97462,118268023~5048976,136310253~353929,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,323482505~7249599,334948415~2373553,337602204~2635002,340372884~5276159,349211446~4192303,358137084~5551677,367576613~5297086,396876058~2804493,406847488~9111979] 2024-03-07T03:49:19.354 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 19 diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.354 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:19.360 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,406847488~9111979] 2024-03-07T03:49:19.360 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 15 diff [15396897~3963058,35879825~1733789,181822460~3589030,197418188~7031285,221858299~3493903,265268048~2824484,323482505~2351426,334948415~2373553,337602204~2635002,367576613~5297086] 2024-03-07T03:49:19.360 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [15396897~3963058,35879825~1733789,181822460~3589030,197418188~7031285,221858299~3493903,265268048~2824484,323482505~2351426,334948415~2373553,337602204~2635002,367576613~5297086] 2024-03-07T03:49:19.366 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [15396897~3963058,35550188~2063426,181822460~3589030,197132288~7317185,221858299~3493903,265268048~2824484,323482505~2351426,334948415~2373553,337602204~2635002,367576613~5297086] 2024-03-07T03:49:19.367 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 16 diff [3099290~416434,15396897~3963058,35879825~1733789,78357669~8827934,118268023~5048976,162665439~6006638,181822460~3589030,197418188~7031285,221858299~3493903,233865617~936151,263372186~5743786,312404032~1018043,323482505~2351426,334948415~2373553,337602204~2635002,367576613~5297086] 2024-03-07T03:49:19.367 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~3963058,35879825~1733789,78357669~8827934,118268023~5048976,162665439~6006638,181822460~3589030,197418188~7031285,221858299~3493903,233865617~936151,263372186~5743786,312404032~1018043,323482505~2351426,334948415~2373553,337602204~2635002,367576613~5297086] 2024-03-07T03:49:19.373 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~3963058,35550188~2063426,78357669~8827934,118268023~5048976,162665439~6006638,181822460~3589030,197132288~7317185,221858299~3493903,233865617~936151,263372186~5743786,312404032~1018043,323482505~2351426,334948415~2373553,337602204~2635002,367576613~5297086] 2024-03-07T03:49:19.373 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 17 diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,118268023~5048976,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,300469008~3708351,312404032~1018043,323482505~2351426,334948415~2373553,337602204~2635002,349211446~4192303,367576613~5297086,410052091~3098836] 2024-03-07T03:49:19.373 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,118268023~5048976,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,300469008~3708351,312404032~1018043,323482505~2351426,334948415~2373553,337602204~2635002,349211446~4192303,367576613~5297086,410052091~3098836] 2024-03-07T03:49:19.379 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~3963058,35550188~2063426,41065687~5569033,57965719~1995106,73665886~13519717,118268023~5048976,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,221858299~3493903,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,300469008~3708351,312404032~1018043,323482505~2351426,334948415~2373553,337602204~2635002,349211446~4192303,367576613~5297086,406847488~6303439] 2024-03-07T03:49:19.379 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 18 diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,112259179~97462,118268023~5048976,136310253~353929,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,323482505~2351426,334948415~2373553,337602204~2635002,343434520~2214523,349211446~4192303,367576613~5297086,409873736~6085731] 2024-03-07T03:49:19.379 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,15396897~3963058,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,112259179~97462,118268023~5048976,136310253~353929,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,323482505~2351426,334948415~2373553,337602204~2635002,343434520~2214523,349211446~4192303,367576613~5297086,409873736~6085731] 2024-03-07T03:49:19.385 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,15396897~3963058,35550188~2063426,41065687~5569033,57965719~1995106,73665886~13519717,112259179~97462,118268023~5048976,136310253~353929,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,221858299~3493903,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,323482505~2351426,334948415~2373553,337602204~2635002,340372884~5276159,349211446~4192303,367576613~5297086,406847488~9111979] 2024-03-07T03:49:19.385 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 19 diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:19.385 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:19.391 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,367576613~5297086,406847488~9111979] 2024-03-07T03:49:19.391 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 16 diff [3099290~416434,78357669~8827934,118268023~5048976,162665439~6006638,223024139~2214744,233865617~936151,263372186~5743786,312404032~1018043,371688229~1167625] 2024-03-07T03:49:19.391 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,78357669~8827934,118268023~5048976,162665439~6006638,223024139~2214744,233865617~936151,263372186~5743786,312404032~1018043,371688229~1167625] 2024-03-07T03:49:19.398 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,78357669~8827934,118268023~5048976,162665439~6006638,223024139~2214744,233865617~936151,263372186~5743786,312404032~1018043,371688229~1167625] 2024-03-07T03:49:19.398 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 17 diff [3099290~416434,41065687~5569033,57965719~1995106,73665886~13519717,118268023~5048976,162665439~6006638,179913275~579688,223024139~2214744,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,300469008~3708351,312404032~1018043,349211446~4192303,371688229~1167625,410052091~3098836] 2024-03-07T03:49:19.398 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,41065687~5569033,57965719~1995106,73665886~13519717,118268023~5048976,162665439~6006638,179913275~579688,223024139~2214744,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,300469008~3708351,312404032~1018043,349211446~4192303,371688229~1167625,410052091~3098836] 2024-03-07T03:49:19.403 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,41065687~5569033,57965719~1995106,73665886~13519717,118268023~5048976,162665439~6006638,179913275~579688,223024139~2214744,233865617~936151,252711974~4792318,263372186~5743786,277702863~5379502,300469008~3708351,312404032~1018043,349211446~4192303,371688229~1167625,406847488~6303439] 2024-03-07T03:49:19.403 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 18 diff [3099290~416434,41065687~5569033,57965719~1995106,73665886~13519717,112259179~97462,118268023~5048976,136310253~353929,162665439~6006638,179913275~579688,200312889~3020170,223024139~2214744,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,343434520~2214523,349211446~4192303,371688229~1167625,409873736~6085731] 2024-03-07T03:49:19.403 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [3099290~416434,41065687~5569033,57965719~1995106,73665886~13519717,112259179~97462,118268023~5048976,136310253~353929,162665439~6006638,179913275~579688,200312889~3020170,223024139~2214744,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,343434520~2214523,349211446~4192303,371688229~1167625,409873736~6085731] 2024-03-07T03:49:19.410 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [3099290~416434,41065687~5569033,57965719~1995106,73665886~13519717,112259179~97462,118268023~5048976,136310253~353929,162665439~6006638,179913275~579688,200312889~3020170,223024139~2214744,233865617~5872767,252711974~4792318,263372186~5743786,274480103~1473964,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,340372884~5276159,349211446~4192303,371688229~1167625,406847488~9111979] 2024-03-07T03:49:19.410 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 19 diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:19.410 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:19.416 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,340372884~5276159,348686973~4716776,371688229~1167625,406847488~9111979] 2024-03-07T03:49:19.416 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to 17 diff [41065687~5569033,57965719~1995106,73665886~6014867,179913275~579688,252711974~4792318,264307050~3516640,277702863~5379502,300469008~3708351,349211446~4192303,410052091~3098836] 2024-03-07T03:49:19.417 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [41065687~5569033,57965719~1995106,73665886~6014867,179913275~579688,252711974~4792318,264307050~3516640,277702863~5379502,300469008~3708351,349211446~4192303,410052091~3098836] 2024-03-07T03:49:19.422 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [41065687~5569033,57965719~1995106,73665886~6014867,179913275~579688,252711974~4792318,264307050~3516640,277702863~5379502,300469008~3708351,349211446~4192303,406847488~6303439] 2024-03-07T03:49:19.422 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to 18 diff [41065687~5569033,57965719~1995106,73665886~6014867,112259179~97462,136310253~353929,179913275~579688,200312889~3020170,234565146~5173238,252711974~4792318,264307050~3516640,274480103~1473964,277702863~10599732,300469008~9085848,321502491~1021206,343434520~2214523,349211446~4192303,409873736~6085731] 2024-03-07T03:49:19.422 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [41065687~5569033,57965719~1995106,73665886~6014867,112259179~97462,136310253~353929,179913275~579688,200312889~3020170,234565146~5173238,252711974~4792318,264307050~3516640,274480103~1473964,277702863~10599732,300469008~9085848,321502491~1021206,343434520~2214523,349211446~4192303,409873736~6085731] 2024-03-07T03:49:19.428 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [41065687~5569033,57965719~1995106,73665886~6014867,112259179~97462,136310253~353929,179913275~579688,200312889~3020170,234565146~5173238,252711974~4792318,264307050~3516640,274480103~1473964,277702863~10599732,300469008~9085848,321502491~1021206,340372884~5276159,349211446~4192303,406847488~9111979] 2024-03-07T03:49:19.428 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to 19 diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,409873736~6085731] 2024-03-07T03:49:19.428 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,409873736~6085731] 2024-03-07T03:49:19.435 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,340372884~5276159,348686973~4716776,406847488~9111979] 2024-03-07T03:49:19.435 INFO:tasks.workunit.client.0.smithi148.stdout:from 17 to 18 diff [112259179~97462,136310253~353929,200312889~3020170,234565146~5173238,274480103~1473964,282881200~5421395,303919958~5634898,321502491~1021206,343434520~2214523,409873736~6085731] 2024-03-07T03:49:19.435 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [112259179~97462,136310253~353929,200312889~3020170,234565146~5173238,274480103~1473964,282881200~5421395,303919958~5634898,321502491~1021206,343434520~2214523,409873736~6085731] 2024-03-07T03:49:19.441 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [112259179~97462,136310253~353929,200312889~3020170,234565146~5173238,274480103~1473964,282881200~5421395,303919958~5634898,321502491~1021206,340372884~5276159,409873736~6085731] 2024-03-07T03:49:19.441 INFO:tasks.workunit.client.0.smithi148.stdout:from 17 to 19 diff [1002676~3282513,10855115~389240,16821258~2887116,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,200312889~3020170,224437494~4921290,234565146~5173238,273147935~2806132,282591780~5710815,303919958~5634898,321502491~1021206,322725949~3038941,343434520~2214523,348686973~2943634,409873736~6085731] 2024-03-07T03:49:19.441 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,200312889~3020170,224437494~4921290,234565146~5173238,273147935~2806132,282591780~5710815,303919958~5634898,321502491~1021206,322725949~3038941,343434520~2214523,348686973~2943634,409873736~6085731] 2024-03-07T03:49:19.447 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,200312889~3020170,224437494~4921290,234565146~5173238,273147935~2806132,282591780~5710815,303919958~5634898,321502491~1021206,322725949~3038941,340372884~5276159,348686973~2943634,409873736~6085731] 2024-03-07T03:49:19.447 INFO:tasks.workunit.client.0.smithi148.stdout:from 18 to 19 diff [1002676~3282513,10855115~389240,16821258~2887116,102509939~1743999,137114000~4310637,224437494~4921290,273147935~1987249,282591780~1737980,322725949~3038941,348686973~2943634] 2024-03-07T03:49:19.447 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,102509939~1743999,137114000~4310637,224437494~4921290,273147935~1987249,282591780~1737980,322725949~3038941,348686973~2943634] 2024-03-07T03:49:19.453 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,102509939~1743999,137114000~4310637,224437494~4921290,273147935~1987249,282591780~1737980,322725949~3038941,348686973~2943634] 2024-03-07T03:49:19.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:19 smithi148 bash[79819]: cluster 2024-03-07T03:49:18.072589+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2024-03-07T03:49:19.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:19 smithi148 bash[82222]: cluster 2024-03-07T03:49:18.072589+0000 mon.a (mon.0) 1856 : cluster 0 osdmap e1018: 8 total, 8 up, 8 in 2024-03-07T03:49:20.091 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:20.091 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:20.140 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:20.141 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:20.141 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:20.155 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:20.155 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.155 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.168 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:20.168 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.168 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.180 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:20.180 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.180 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.190 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:20.190 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.191 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.199 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:20.199 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.199 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:20.208 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:20.208 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.208 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.216 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:20.216 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.216 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.223 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:20.223 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.223 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.230 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:20.230 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.230 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.237 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:20.237 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.237 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:20.244 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:20.244 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:20.244 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:20.250 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:20.250 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:20.250 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:20.257 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,406847488~9111979] 2024-03-07T03:49:20.257 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:20.258 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:20.264 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,367576613~5297086,406847488~9111979] 2024-03-07T03:49:20.264 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to HEAD diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:20.264 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:20.271 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,340372884~5276159,348686973~4716776,371688229~1167625,406847488~9111979] 2024-03-07T03:49:20.271 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to HEAD diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,409873736~6085731] 2024-03-07T03:49:20.271 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,409873736~6085731] 2024-03-07T03:49:20.277 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,340372884~5276159,348686973~4716776,406847488~9111979] 2024-03-07T03:49:20.277 INFO:tasks.workunit.client.0.smithi148.stdout:from 17 to HEAD diff [1002676~3282513,10855115~389240,16821258~2887116,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,200312889~3020170,224437494~4921290,234565146~5173238,273147935~2806132,282591780~5710815,303919958~5634898,321502491~1021206,322725949~3038941,343434520~2214523,348686973~2943634,409873736~6085731] 2024-03-07T03:49:20.277 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,200312889~3020170,224437494~4921290,234565146~5173238,273147935~2806132,282591780~5710815,303919958~5634898,321502491~1021206,322725949~3038941,343434520~2214523,348686973~2943634,409873736~6085731] 2024-03-07T03:49:20.283 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,200312889~3020170,224437494~4921290,234565146~5173238,273147935~2806132,282591780~5710815,303919958~5634898,321502491~1021206,322725949~3038941,340372884~5276159,348686973~2943634,409873736~6085731] 2024-03-07T03:49:20.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:20 smithi195 bash[66542]: cluster 2024-03-07T03:49:19.237340+0000 mgr.y (mgr.34101) 1061 : cluster 0 pgmap v1698: 105 pgs: 105 active+clean; 854 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 245 KiB/s rd, 34 MiB/s wr, 1.01k op/s 2024-03-07T03:49:20.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:20 smithi148 bash[79819]: cluster 2024-03-07T03:49:19.237340+0000 mgr.y (mgr.34101) 1061 : cluster 0 pgmap v1698: 105 pgs: 105 active+clean; 854 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 245 KiB/s rd, 34 MiB/s wr, 1.01k op/s 2024-03-07T03:49:20.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:20 smithi148 bash[82222]: cluster 2024-03-07T03:49:19.237340+0000 mgr.y (mgr.34101) 1061 : cluster 0 pgmap v1698: 105 pgs: 105 active+clean; 854 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 245 KiB/s rd, 34 MiB/s wr, 1.01k op/s 2024-03-07T03:49:21.111 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:21.111 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:21.133 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:21.133 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:21.133 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:21.145 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:21.145 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.145 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.155 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:21.155 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.156 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.165 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:21.166 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.166 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.174 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:21.174 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.174 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.182 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:21.182 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.182 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:21.190 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:21.190 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.190 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.197 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:21.197 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.197 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.205 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:21.205 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.205 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.210 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:21.210 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.210 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.216 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:21.217 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.217 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:21.223 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:21.223 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:21.223 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:21.229 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:21.229 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:21.229 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:21.236 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,406847488~9111979] 2024-03-07T03:49:21.236 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:21.236 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:21.243 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,367576613~5297086,406847488~9111979] 2024-03-07T03:49:21.243 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to HEAD diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:21.243 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:21.248 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,340372884~5276159,348686973~4716776,371688229~1167625,406847488~9111979] 2024-03-07T03:49:21.249 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to HEAD diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,409873736~6085731] 2024-03-07T03:49:21.249 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,409873736~6085731] 2024-03-07T03:49:21.254 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,41065687~5569033,57965719~1995106,73665886~6014867,102509939~1743999,112259179~97462,136310253~353929,137114000~4310637,179913275~579688,200312889~3020170,224437494~4921290,234565146~5173238,252711974~4792318,264307050~3516640,273147935~2806132,277702863~10599732,300469008~9085848,321502491~1021206,322725949~3038941,340372884~5276159,348686973~4716776,406847488~9111979] 2024-03-07T03:49:21.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:21 smithi148 bash[79819]: cluster 2024-03-07T03:49:20.079264+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2024-03-07T03:49:21.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:21 smithi148 bash[82222]: cluster 2024-03-07T03:49:20.079264+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2024-03-07T03:49:21.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:21 smithi195 bash[66542]: cluster 2024-03-07T03:49:20.079264+0000 mon.a (mon.0) 1857 : cluster 0 osdmap e1019: 8 total, 8 up, 8 in 2024-03-07T03:49:22.138 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:22.256 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:22.261 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.262 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.263 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:22.264 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.264 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:22.264 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:22.264 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:22.264 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:22.265 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:22.265 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:22.265 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:22.272 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,406847488~9111979] 2024-03-07T03:49:22.272 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:22.272 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:22.278 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,367576613~5297086,406847488~9111979] 2024-03-07T03:49:22.278 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to HEAD diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:22.278 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,343434520~2214523,348686973~4716776,371688229~1167625,409873736~6085731] 2024-03-07T03:49:22.283 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,16821258~2887116,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,200312889~3020170,223024139~6334645,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3038941,340372884~5276159,348686973~4716776,371688229~1167625,406847488~9111979] 2024-03-07T03:49:22.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:22 smithi148 bash[79819]: cluster 2024-03-07T03:49:21.084279+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2024-03-07T03:49:22.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:22 smithi148 bash[79819]: cluster 2024-03-07T03:49:21.237801+0000 mgr.y (mgr.34101) 1062 : cluster 0 pgmap v1701: 105 pgs: 105 active+clean; 863 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 194 KiB/s rd, 24 MiB/s wr, 1.61k op/s 2024-03-07T03:49:22.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:22 smithi148 bash[82222]: cluster 2024-03-07T03:49:21.084279+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2024-03-07T03:49:22.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:22 smithi148 bash[82222]: cluster 2024-03-07T03:49:21.237801+0000 mgr.y (mgr.34101) 1062 : cluster 0 pgmap v1701: 105 pgs: 105 active+clean; 863 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 194 KiB/s rd, 24 MiB/s wr, 1.61k op/s 2024-03-07T03:49:22.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:22 smithi195 bash[66542]: cluster 2024-03-07T03:49:21.084279+0000 mon.a (mon.0) 1858 : cluster 0 osdmap e1020: 8 total, 8 up, 8 in 2024-03-07T03:49:22.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:22 smithi195 bash[66542]: cluster 2024-03-07T03:49:21.237801+0000 mgr.y (mgr.34101) 1062 : cluster 0 pgmap v1701: 105 pgs: 105 active+clean; 863 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 194 KiB/s rd, 24 MiB/s wr, 1.61k op/s 2024-03-07T03:49:22.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:49:22 smithi195 bash[59134]: ts=2024-03-07T03:49:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:23.140 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:23.140 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:23.176 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:23.176 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:23.176 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:23.186 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:23.186 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.186 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.195 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:23.195 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.195 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.204 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:23.204 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.204 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.212 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:23.212 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.212 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.219 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:23.220 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.220 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:23.227 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:23.227 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.227 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.233 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:23.233 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.233 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.240 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:23.240 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.240 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.246 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:23.246 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.246 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.253 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:23.253 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.254 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:23.259 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:23.259 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:23.259 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:23.266 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:23.266 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:23.266 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:23.272 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,406847488~9111979] 2024-03-07T03:49:23.272 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:23.272 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,367576613~5297086,409873736~6085731] 2024-03-07T03:49:23.278 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,73665886~13519717,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,300469008~9085848,312404032~1018043,321502491~1021206,322725949~3107982,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,367576613~5297086,406847488~9111979] 2024-03-07T03:49:23.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:23 smithi148 bash[79819]: cluster 2024-03-07T03:49:22.111461+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2024-03-07T03:49:23.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:23 smithi148 bash[82222]: cluster 2024-03-07T03:49:22.111461+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2024-03-07T03:49:23.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:23 smithi195 bash[66542]: cluster 2024-03-07T03:49:22.111461+0000 mon.a (mon.0) 1859 : cluster 0 osdmap e1021: 8 total, 8 up, 8 in 2024-03-07T03:49:24.157 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:24.158 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:24.195 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:24.195 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:24.195 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:24.205 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:24.206 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.206 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.215 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:24.215 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.215 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.223 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:24.223 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.224 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.232 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:24.232 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.232 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.239 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:24.240 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.240 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:24.247 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:24.247 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.247 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.254 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:24.254 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.254 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.260 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:24.260 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.260 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.266 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:24.267 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.267 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.273 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:24.273 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.273 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:24.279 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:24.280 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:24.280 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:24.286 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:24.286 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:24.286 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~4311477,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,343434520~2214523,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,409873736~6085731] 2024-03-07T03:49:24.292 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~4311477,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,155324570~457207,162665439~6006638,179913275~579688,181822460~3589030,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~8006155,334948415~2373553,337602204~2635002,340372884~5276159,348686973~4716776,358137084~5551677,367576613~5297086,396876058~2804493,406847488~9111979] 2024-03-07T03:49:24.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:24 smithi148 bash[79819]: cluster 2024-03-07T03:49:23.110870+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2024-03-07T03:49:24.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:24 smithi148 bash[79819]: cluster 2024-03-07T03:49:23.238559+0000 mgr.y (mgr.34101) 1063 : cluster 0 pgmap v1704: 105 pgs: 105 active+clean; 860 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 193 KiB/s rd, 2.4 MiB/s wr, 4.81k op/s 2024-03-07T03:49:24.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:24 smithi148 bash[82222]: cluster 2024-03-07T03:49:23.110870+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2024-03-07T03:49:24.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:24 smithi148 bash[82222]: cluster 2024-03-07T03:49:23.238559+0000 mgr.y (mgr.34101) 1063 : cluster 0 pgmap v1704: 105 pgs: 105 active+clean; 860 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 193 KiB/s rd, 2.4 MiB/s wr, 4.81k op/s 2024-03-07T03:49:24.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:24 smithi195 bash[66542]: cluster 2024-03-07T03:49:23.110870+0000 mon.a (mon.0) 1860 : cluster 0 osdmap e1022: 8 total, 8 up, 8 in 2024-03-07T03:49:24.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:24 smithi195 bash[66542]: cluster 2024-03-07T03:49:23.238559+0000 mgr.y (mgr.34101) 1063 : cluster 0 pgmap v1704: 105 pgs: 105 active+clean; 860 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 193 KiB/s rd, 2.4 MiB/s wr, 4.81k op/s 2024-03-07T03:49:25.164 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:25.164 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:25.200 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:25.200 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:25.200 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:25.210 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:25.210 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.210 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.220 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:25.221 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.221 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.226 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:25.226 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.226 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.234 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:25.234 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.234 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.240 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:25.240 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.241 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:25.247 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:25.247 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.247 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.254 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:25.254 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.254 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.260 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:25.260 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.260 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.266 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:25.267 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.267 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.273 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:25.273 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.273 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:25.278 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:25.278 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:25.279 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~7435944,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,409873736~6085731] 2024-03-07T03:49:25.284 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~7435944,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~7500485,233865617~5872767,252711974~4792318,263372186~5743786,273147935~2806132,277702863~10599732,290524600~5496444,300469008~9085848,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,388807829~4661986,394504318~583478,396876058~2804493,406847488~9111979] 2024-03-07T03:49:25.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:25 smithi148 bash[79819]: cluster 2024-03-07T03:49:24.133731+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2024-03-07T03:49:25.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:25 smithi148 bash[82222]: cluster 2024-03-07T03:49:24.133731+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2024-03-07T03:49:25.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:25 smithi195 bash[66542]: cluster 2024-03-07T03:49:24.133731+0000 mon.a (mon.0) 1861 : cluster 0 osdmap e1023: 8 total, 8 up, 8 in 2024-03-07T03:49:26.189 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:26.189 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:26.224 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:26.225 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:26.225 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:26.234 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:26.234 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.234 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.242 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:26.242 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.243 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.249 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:26.249 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.250 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.257 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:26.257 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.257 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.263 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:26.263 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.264 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:26.270 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:26.270 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.270 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.276 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:26.277 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.277 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.283 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:26.283 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.283 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.289 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:26.289 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.289 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.296 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:26.296 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.296 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,300302651~9252205,312404032~1018043,315737520~6846870,322725949~17511257,343434520~2214523,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:26.302 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,57965719~1995106,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,155324570~457207,155832535~12839542,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,252711974~4792318,263372186~5743786,273147935~2806132,277702863~18318181,297989508~11565348,312404032~1018043,315737520~6846870,322725949~17511257,340372884~5276159,348686973~4716776,358137084~5551677,366918734~5954965,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:26.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:26 smithi148 bash[79819]: cluster 2024-03-07T03:49:25.134941+0000 mon.a (mon.0) 1862 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2024-03-07T03:49:26.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:26 smithi148 bash[79819]: cluster 2024-03-07T03:49:25.239150+0000 mgr.y (mgr.34101) 1064 : cluster 0 pgmap v1707: 105 pgs: 105 active+clean; 855 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 322 KiB/s wr, 4.82k op/s 2024-03-07T03:49:26.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:26 smithi148 bash[82222]: cluster 2024-03-07T03:49:25.134941+0000 mon.a (mon.0) 1862 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2024-03-07T03:49:26.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:26 smithi148 bash[82222]: cluster 2024-03-07T03:49:25.239150+0000 mgr.y (mgr.34101) 1064 : cluster 0 pgmap v1707: 105 pgs: 105 active+clean; 855 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 322 KiB/s wr, 4.82k op/s 2024-03-07T03:49:26.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:26 smithi195 bash[66542]: cluster 2024-03-07T03:49:25.134941+0000 mon.a (mon.0) 1862 : cluster 0 osdmap e1024: 8 total, 8 up, 8 in 2024-03-07T03:49:26.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:26 smithi195 bash[66542]: cluster 2024-03-07T03:49:25.239150+0000 mgr.y (mgr.34101) 1064 : cluster 0 pgmap v1707: 105 pgs: 105 active+clean; 855 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 148 KiB/s rd, 322 KiB/s wr, 4.82k op/s 2024-03-07T03:49:27.187 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.187 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.231 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:27.231 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.232 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.240 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:27.240 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.240 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.248 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.248 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.248 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.256 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.256 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.256 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.264 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.264 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.264 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.270 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.270 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.270 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.277 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.277 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.277 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.283 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:27.283 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.283 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.289 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:27.289 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.290 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.295 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:27.295 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.295 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,31904019~2211031,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,179913275~579688,180903040~4508450,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.301 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,31904019~2211031,35550188~2063426,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,179913275~579688,180903040~4508450,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,249454878~8049414,263372186~5743786,273147935~2806132,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:27.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:27 smithi148 bash[79819]: cluster 2024-03-07T03:49:26.160593+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2024-03-07T03:49:27.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:49:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:49:27] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-03-07T03:49:27.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:27 smithi148 bash[82222]: cluster 2024-03-07T03:49:26.160593+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2024-03-07T03:49:27.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:27 smithi195 bash[66542]: cluster 2024-03-07T03:49:26.160593+0000 mon.a (mon.0) 1863 : cluster 0 osdmap e1025: 8 total, 8 up, 8 in 2024-03-07T03:49:27.814 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.814 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.860 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:27.861 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.861 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:27.869 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:27.869 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.869 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.878 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.878 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.878 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.886 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.886 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.886 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.892 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.892 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.893 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.900 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.900 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.900 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:27.906 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:27.906 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.906 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.912 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:27.912 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.913 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.919 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:27.919 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.919 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:27.925 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,69963884~17221719,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,221858299~11027565,233865617~5872767,243876757~4426093,248347320~9156972,263372186~5743786,273147935~3642109,277702863~19258111,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,358137084~14736615,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:28.161 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:28 smithi195 bash[66542]: cluster 2024-03-07T03:49:27.158580+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2024-03-07T03:49:28.161 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:28 smithi195 bash[66542]: cluster 2024-03-07T03:49:27.239839+0000 mgr.y (mgr.34101) 1065 : cluster 0 pgmap v1710: 105 pgs: 105 active+clean; 844 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 1.15k op/s 2024-03-07T03:49:28.162 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:28 smithi195 bash[66542]: audit 2024-03-07T03:49:27.351109+0000 mon.a (mon.0) 1865 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:28.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[79819]: cluster 2024-03-07T03:49:27.158580+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2024-03-07T03:49:28.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[79819]: cluster 2024-03-07T03:49:27.239839+0000 mgr.y (mgr.34101) 1065 : cluster 0 pgmap v1710: 105 pgs: 105 active+clean; 844 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 1.15k op/s 2024-03-07T03:49:28.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[79819]: audit 2024-03-07T03:49:27.351109+0000 mon.a (mon.0) 1865 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:28.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[79819]: cluster 2024-03-07T03:49:27.777875+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2024-03-07T03:49:28.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[82222]: cluster 2024-03-07T03:49:27.158580+0000 mon.a (mon.0) 1864 : cluster 0 osdmap e1026: 8 total, 8 up, 8 in 2024-03-07T03:49:28.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[82222]: cluster 2024-03-07T03:49:27.239839+0000 mgr.y (mgr.34101) 1065 : cluster 0 pgmap v1710: 105 pgs: 105 active+clean; 844 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 10 KiB/s rd, 1.2 KiB/s wr, 1.15k op/s 2024-03-07T03:49:28.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[82222]: audit 2024-03-07T03:49:27.351109+0000 mon.a (mon.0) 1865 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:28.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:28 smithi148 bash[82222]: cluster 2024-03-07T03:49:27.777875+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2024-03-07T03:49:28.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:28 smithi195 bash[66542]: cluster 2024-03-07T03:49:27.777875+0000 mon.a (mon.0) 1866 : cluster 0 osdmap e1027: 8 total, 8 up, 8 in 2024-03-07T03:49:28.807 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:28.807 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:28.855 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:28.855 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:28.855 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:28.863 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:28.863 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.863 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.871 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:28.871 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.871 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.878 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:28.878 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.878 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.889 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:28.889 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.889 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.896 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:28.896 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.896 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:28.901 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:28.901 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:28.902 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:28.908 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:28.908 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:28.908 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~5872767,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,312404032~1018043,315737520~24499686,343434520~2214523,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:28.914 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,112259179~97462,112841617~4261226,118268023~5048976,136310253~353929,137114000~4310637,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~5872767,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,312404032~1018043,315737520~24499686,340372884~5276159,348686973~4716776,357034236~15839463,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:29.820 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:29.820 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:29.863 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:29.863 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:29.863 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:29.872 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:29.872 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.872 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.879 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:29.880 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.880 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.886 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:29.886 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.886 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.893 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:29.893 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.894 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.900 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:29.900 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.900 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:29.906 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:29.906 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:29.907 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,56594107~6132924,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,220902876~11982988,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28390243,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,409873736~6085731] 2024-03-07T03:49:29.913 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,56594107~6132924,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7906202,143458483~109788,147091666~5234760,154546637~14125440,173783257~4865208,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,220902876~11982988,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~28390243,340372884~5276159,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~583478,396810984~4315416,406847488~9111979] 2024-03-07T03:49:30.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:29 smithi148 bash[79819]: cluster 2024-03-07T03:49:28.779982+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2024-03-07T03:49:30.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:29 smithi148 bash[82222]: cluster 2024-03-07T03:49:28.779982+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2024-03-07T03:49:30.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:29 smithi195 bash[66542]: cluster 2024-03-07T03:49:28.779982+0000 mon.a (mon.0) 1867 : cluster 0 osdmap e1028: 8 total, 8 up, 8 in 2024-03-07T03:49:30.829 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:30.830 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:30.877 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:30.877 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:30.877 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:30.885 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:30.885 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.885 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.894 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:30.894 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.894 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.901 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:30.901 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.901 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.908 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:30.908 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.909 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.915 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:30.915 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.915 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~3282513,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,41065687~5569033,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~3865095,263372186~5743786,271980343~4809701,277702863~19258111,300302651~9252205,311846963~28525921,343434520~2214523,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:30.922 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~3282513,9346188~1898167,15396897~9284123,27243122~10370492,41065687~5569033,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,218437948~14447916,233865617~7822493,243876757~4426093,248347320~14145721,263372186~5743786,271980343~4809701,276824064~20136910,297989508~11565348,311846963~33802080,348686973~4716776,357034236~18485837,375723852~2962468,386954373~6992617,394264576~6861824,406847488~9111979] 2024-03-07T03:49:31.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:30 smithi148 bash[79819]: cluster 2024-03-07T03:49:29.240743+0000 mgr.y (mgr.34101) 1066 : cluster 0 pgmap v1713: 105 pgs: 105 active+clean; 785 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 5.7 KiB/s wr, 1.78k op/s 2024-03-07T03:49:31.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:30 smithi148 bash[79819]: cluster 2024-03-07T03:49:29.784937+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2024-03-07T03:49:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:30 smithi148 bash[82222]: cluster 2024-03-07T03:49:29.240743+0000 mgr.y (mgr.34101) 1066 : cluster 0 pgmap v1713: 105 pgs: 105 active+clean; 785 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 5.7 KiB/s wr, 1.78k op/s 2024-03-07T03:49:31.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:30 smithi148 bash[82222]: cluster 2024-03-07T03:49:29.784937+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2024-03-07T03:49:31.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:30 smithi195 bash[66542]: cluster 2024-03-07T03:49:29.240743+0000 mgr.y (mgr.34101) 1066 : cluster 0 pgmap v1713: 105 pgs: 105 active+clean; 785 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 116 KiB/s rd, 5.7 KiB/s wr, 1.78k op/s 2024-03-07T03:49:31.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:30 smithi195 bash[66542]: cluster 2024-03-07T03:49:29.784937+0000 mon.a (mon.0) 1868 : cluster 0 osdmap e1029: 8 total, 8 up, 8 in 2024-03-07T03:49:31.839 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:31.839 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:31.885 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:31.885 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:31.885 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:31.893 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:31.894 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.894 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.902 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:31.902 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.902 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.909 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:31.909 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.909 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.917 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:31.917 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.917 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,102509939~1743999,107640648~1371632,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~19258111,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:31.923 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,102509939~1743999,106662570~2349710,112259179~97462,112841617~4261226,118268023~5048976,135426417~7976350,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~20136910,297989508~55414241,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:32.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:31 smithi195 bash[66542]: cluster 2024-03-07T03:49:30.798546+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2024-03-07T03:49:32.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:31 smithi148 bash[79819]: cluster 2024-03-07T03:49:30.798546+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2024-03-07T03:49:32.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:31 smithi148 bash[82222]: cluster 2024-03-07T03:49:30.798546+0000 mon.a (mon.0) 1869 : cluster 0 osdmap e1030: 8 total, 8 up, 8 in 2024-03-07T03:49:32.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:49:32 smithi195 bash[59134]: ts=2024-03-07T03:49:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:32.816 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:32.816 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:32.853 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:32.853 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:32.853 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:32.862 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:32.862 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:32.862 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:32.869 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:32.869 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:32.869 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:32.876 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:32.876 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:32.876 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,54508011~8219020,66725534~1527286,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~1285898,158985649~9686428,172622639~1160618,175143458~3505007,179913275~579688,180903040~4508450,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233865617~7822493,243876757~4426093,248347320~9156972,258627946~10488026,271980343~4809701,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:32.883 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~8885984,54508011~8219020,62914560~5338260,68844351~18341252,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,147091666~5234760,154546637~14125440,171966464~6682001,179913275~579688,180903040~4508450,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233865617~7822493,243876757~4426093,248347320~20768652,271980343~4809701,276824064~76579685,357034236~18485837,375723852~2962468,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:33.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:32 smithi195 bash[66542]: cluster 2024-03-07T03:49:31.241443+0000 mgr.y (mgr.34101) 1067 : cluster 0 pgmap v1716: 105 pgs: 105 active+clean; 766 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 7.2 KiB/s wr, 1.64k op/s 2024-03-07T03:49:33.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:32 smithi195 bash[66542]: cluster 2024-03-07T03:49:31.818604+0000 mon.a (mon.0) 1870 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2024-03-07T03:49:33.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:32 smithi195 bash[66542]: cluster 2024-03-07T03:49:32.780788+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2024-03-07T03:49:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:32 smithi148 bash[79819]: cluster 2024-03-07T03:49:31.241443+0000 mgr.y (mgr.34101) 1067 : cluster 0 pgmap v1716: 105 pgs: 105 active+clean; 766 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 7.2 KiB/s wr, 1.64k op/s 2024-03-07T03:49:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:32 smithi148 bash[79819]: cluster 2024-03-07T03:49:31.818604+0000 mon.a (mon.0) 1870 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2024-03-07T03:49:33.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:32 smithi148 bash[79819]: cluster 2024-03-07T03:49:32.780788+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2024-03-07T03:49:33.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:32 smithi148 bash[82222]: cluster 2024-03-07T03:49:31.241443+0000 mgr.y (mgr.34101) 1067 : cluster 0 pgmap v1716: 105 pgs: 105 active+clean; 766 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 7.2 KiB/s wr, 1.64k op/s 2024-03-07T03:49:33.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:32 smithi148 bash[82222]: cluster 2024-03-07T03:49:31.818604+0000 mon.a (mon.0) 1870 : cluster 0 osdmap e1031: 8 total, 8 up, 8 in 2024-03-07T03:49:33.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:32 smithi148 bash[82222]: cluster 2024-03-07T03:49:32.780788+0000 mon.a (mon.0) 1871 : cluster 0 osdmap e1032: 8 total, 8 up, 8 in 2024-03-07T03:49:33.858 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:33.858 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:33.901 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:33.901 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:33.901 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:33.910 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:33.910 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:33.910 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:33.919 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:33.919 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:33.919 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~7082767,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~10268032,187362711~1171839,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~9969229,357034236~18485837,375723852~2962468,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:33.927 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [1002676~7082767,9346188~1898167,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,112259179~97462,112841617~12000067,131400499~12002268,143458483~109788,146733854~21938223,171966464~13445026,187362711~1171839,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76579685,357034236~18485837,375723852~2962468,379453072~2944524,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:34.867 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:34.867 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:34.909 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:34.909 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:34.910 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:34.919 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:34.919 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:34.919 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~9284123,27243122~8307066,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,357034236~22418836,380459900~1937696,386954373~6992617,394504318~6622082,409873736~6085731] 2024-03-07T03:49:34.927 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~12098108,27243122~10370492,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,357034236~25363360,385875968~8071022,394264576~6861824,406847488~9111979] 2024-03-07T03:49:35.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:34 smithi148 bash[79819]: cluster 2024-03-07T03:49:33.242383+0000 mgr.y (mgr.34101) 1068 : cluster 0 pgmap v1719: 105 pgs: 105 active+clean; 683 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 6.5 KiB/s wr, 895 op/s 2024-03-07T03:49:35.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:34 smithi148 bash[79819]: cluster 2024-03-07T03:49:33.834000+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2024-03-07T03:49:35.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:34 smithi148 bash[82222]: cluster 2024-03-07T03:49:33.242383+0000 mgr.y (mgr.34101) 1068 : cluster 0 pgmap v1719: 105 pgs: 105 active+clean; 683 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 6.5 KiB/s wr, 895 op/s 2024-03-07T03:49:35.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:34 smithi148 bash[82222]: cluster 2024-03-07T03:49:33.834000+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2024-03-07T03:49:35.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:34 smithi195 bash[66542]: cluster 2024-03-07T03:49:33.242383+0000 mgr.y (mgr.34101) 1068 : cluster 0 pgmap v1719: 105 pgs: 105 active+clean; 683 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 6.5 KiB/s wr, 895 op/s 2024-03-07T03:49:35.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:34 smithi195 bash[66542]: cluster 2024-03-07T03:49:33.834000+0000 mon.a (mon.0) 1872 : cluster 0 osdmap e1033: 8 total, 8 up, 8 in 2024-03-07T03:49:35.871 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:35.871 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:35.913 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:35.913 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:35.914 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146733854~9098681,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~12978355,233433683~8254427,243876757~4426093,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:35.922 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146733854~21938223,171966464~16568086,197132288~7317185,205520896~7904290,213909504~18976360,233433683~8254427,243876757~4426093,248347320~28442724,276824064~76648839,356612098~25785498,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:36.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:35 smithi148 bash[79819]: cluster 2024-03-07T03:49:34.842078+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2024-03-07T03:49:36.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:35 smithi148 bash[82222]: cluster 2024-03-07T03:49:34.842078+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2024-03-07T03:49:36.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:35 smithi195 bash[66542]: cluster 2024-03-07T03:49:34.842078+0000 mon.a (mon.0) 1873 : cluster 0 osdmap e1034: 8 total, 8 up, 8 in 2024-03-07T03:49:36.883 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:36.883 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [1002676~8343512,10855115~389240,15396897~20153291,35879825~1733789,38735630~7899090,47457196~15269835,66725534~1527286,68844351~22511106,93923603~1285999,101146541~1069076,102509939~1743999,107640648~1371632,111365223~15196568,131400499~12002268,143458483~109788,146288090~9544445,158985649~9686428,172622639~1160618,175143458~13391092,197418188~7031285,207908122~5517064,217208288~1229660,219907509~28395341,248347320~9156972,258627946~18162098,277702863~20286645,300302651~40070233,343434520~10038383,356612098~22840974,380459900~1937696,386954373~6992617,394504318~6622082,404168647~1959084,409873736~6085731] 2024-03-07T03:49:36.916 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~11244355,12582912~25030702,37748736~24978295,62914560~28440897,93923603~1285999,101146541~1069076,102509939~1743999,106662570~2349710,109051904~17509887,131400499~12002268,143458483~109788,146288090~22383987,171966464~16568086,197132288~7317185,205520896~7904290,213909504~34393346,248347320~28442724,276824064~76648839,356515840~25881756,385875968~8071022,394264576~6861824,402653184~3474547,406847488~9111979] 2024-03-07T03:49:37.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:36 smithi148 bash[79819]: cluster 2024-03-07T03:49:35.243123+0000 mgr.y (mgr.34101) 1069 : cluster 0 pgmap v1722: 105 pgs: 105 active+clean; 629 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 6.2 KiB/s wr, 887 op/s 2024-03-07T03:49:37.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:36 smithi148 bash[79819]: cluster 2024-03-07T03:49:35.851577+0000 mon.a (mon.0) 1874 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2024-03-07T03:49:37.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:36 smithi148 bash[82222]: cluster 2024-03-07T03:49:35.243123+0000 mgr.y (mgr.34101) 1069 : cluster 0 pgmap v1722: 105 pgs: 105 active+clean; 629 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 6.2 KiB/s wr, 887 op/s 2024-03-07T03:49:37.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:36 smithi148 bash[82222]: cluster 2024-03-07T03:49:35.851577+0000 mon.a (mon.0) 1874 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2024-03-07T03:49:37.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:36 smithi195 bash[66542]: cluster 2024-03-07T03:49:35.243123+0000 mgr.y (mgr.34101) 1069 : cluster 0 pgmap v1722: 105 pgs: 105 active+clean; 629 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 6.2 KiB/s wr, 887 op/s 2024-03-07T03:49:37.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:36 smithi195 bash[66542]: cluster 2024-03-07T03:49:35.851577+0000 mon.a (mon.0) 1874 : cluster 0 osdmap e1035: 8 total, 8 up, 8 in 2024-03-07T03:49:37.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:49:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:49:37] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:49:37.841 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (40068 ms) 2024-03-07T03:49:37.841 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-03-07T03:49:37.842 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:38.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:37 smithi148 bash[79819]: cluster 2024-03-07T03:49:36.858307+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2024-03-07T03:49:38.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:37 smithi148 bash[79819]: cluster 2024-03-07T03:49:37.784570+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2024-03-07T03:49:38.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:37 smithi148 bash[82222]: cluster 2024-03-07T03:49:36.858307+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2024-03-07T03:49:38.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:37 smithi148 bash[82222]: cluster 2024-03-07T03:49:37.784570+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2024-03-07T03:49:38.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:37 smithi195 bash[66542]: cluster 2024-03-07T03:49:36.858307+0000 mon.a (mon.0) 1875 : cluster 0 osdmap e1036: 8 total, 8 up, 8 in 2024-03-07T03:49:38.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:37 smithi195 bash[66542]: cluster 2024-03-07T03:49:37.784570+0000 mon.a (mon.0) 1876 : cluster 0 osdmap e1037: 8 total, 8 up, 8 in 2024-03-07T03:49:39.146 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1305 ms) 2024-03-07T03:49:39.146 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-03-07T03:49:39.146 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:39.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:38 smithi148 bash[79819]: cluster 2024-03-07T03:49:37.243837+0000 mgr.y (mgr.34101) 1070 : cluster 0 pgmap v1725: 105 pgs: 105 active+clean; 595 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.2 KiB/s wr, 298 op/s 2024-03-07T03:49:39.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:38 smithi148 bash[82222]: cluster 2024-03-07T03:49:37.243837+0000 mgr.y (mgr.34101) 1070 : cluster 0 pgmap v1725: 105 pgs: 105 active+clean; 595 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.2 KiB/s wr, 298 op/s 2024-03-07T03:49:39.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:38 smithi195 bash[66542]: cluster 2024-03-07T03:49:37.243837+0000 mgr.y (mgr.34101) 1070 : cluster 0 pgmap v1725: 105 pgs: 105 active+clean; 595 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 KiB/s rd, 1.2 KiB/s wr, 298 op/s 2024-03-07T03:49:40.181 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [168259~88678,3431161~51652,3921304~34404,4578840~57432,9727507~89675,9891290~84571,12197244~97423,12802743~29271,15447105~34796,16108807~6151] to clone 2024-03-07T03:49:40.183 INFO:tasks.workunit.client.0.smithi148.stdout: diff was [0~3955708,4194304~441968,8388608~3906059,12582912~3532046] 2024-03-07T03:49:40.192 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1046 ms) 2024-03-07T03:49:40.192 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-03-07T03:49:40.193 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:40.244 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [23372~27576,294313~55359,1713957~45536,3108562~45522,8464835~15675,13908362~39032,16127175~92613,17116619~61187,18216575~39076,18764290~19385] 2024-03-07T03:49:40.271 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (79 ms) 2024-03-07T03:49:40.271 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-03-07T03:49:40.271 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:40.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:39 smithi148 bash[79819]: cluster 2024-03-07T03:49:38.900837+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2024-03-07T03:49:40.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:39 smithi148 bash[79819]: cluster 2024-03-07T03:49:39.244805+0000 mgr.y (mgr.34101) 1071 : cluster 0 pgmap v1728: 105 pgs: 105 active+clean; 473 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 3.0 MiB/s wr, 356 op/s 2024-03-07T03:49:40.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:39 smithi148 bash[82222]: cluster 2024-03-07T03:49:38.900837+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2024-03-07T03:49:40.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:39 smithi148 bash[82222]: cluster 2024-03-07T03:49:39.244805+0000 mgr.y (mgr.34101) 1071 : cluster 0 pgmap v1728: 105 pgs: 105 active+clean; 473 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 3.0 MiB/s wr, 356 op/s 2024-03-07T03:49:40.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:39 smithi195 bash[66542]: cluster 2024-03-07T03:49:38.900837+0000 mon.a (mon.0) 1877 : cluster 0 osdmap e1038: 8 total, 8 up, 8 in 2024-03-07T03:49:40.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:39 smithi195 bash[66542]: cluster 2024-03-07T03:49:39.244805+0000 mgr.y (mgr.34101) 1071 : cluster 0 pgmap v1728: 105 pgs: 105 active+clean; 473 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 57 KiB/s rd, 3.0 MiB/s wr, 356 op/s 2024-03-07T03:49:41.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:40 smithi148 bash[79819]: cluster 2024-03-07T03:49:39.910313+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2024-03-07T03:49:41.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:40 smithi148 bash[82222]: cluster 2024-03-07T03:49:39.910313+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2024-03-07T03:49:41.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:40 smithi195 bash[66542]: cluster 2024-03-07T03:49:39.910313+0000 mon.a (mon.0) 1878 : cluster 0 osdmap e1039: 8 total, 8 up, 8 in 2024-03-07T03:49:42.179 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1907 ms) 2024-03-07T03:49:42.179 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-03-07T03:49:42.179 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:42.288 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (110 ms) 2024-03-07T03:49:42.289 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-03-07T03:49:42.289 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:42.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:41 smithi148 bash[79819]: cluster 2024-03-07T03:49:40.919829+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2024-03-07T03:49:42.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:41 smithi148 bash[79819]: cluster 2024-03-07T03:49:41.245523+0000 mgr.y (mgr.34101) 1072 : cluster 0 pgmap v1731: 105 pgs: 105 active+clean; 462 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 281 op/s 2024-03-07T03:49:42.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:41 smithi148 bash[82222]: cluster 2024-03-07T03:49:40.919829+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2024-03-07T03:49:42.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:41 smithi148 bash[82222]: cluster 2024-03-07T03:49:41.245523+0000 mgr.y (mgr.34101) 1072 : cluster 0 pgmap v1731: 105 pgs: 105 active+clean; 462 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 281 op/s 2024-03-07T03:49:42.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:41 smithi195 bash[66542]: cluster 2024-03-07T03:49:40.919829+0000 mon.a (mon.0) 1879 : cluster 0 osdmap e1040: 8 total, 8 up, 8 in 2024-03-07T03:49:42.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:41 smithi195 bash[66542]: cluster 2024-03-07T03:49:41.245523+0000 mgr.y (mgr.34101) 1072 : cluster 0 pgmap v1731: 105 pgs: 105 active+clean; 462 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 281 op/s 2024-03-07T03:49:42.409 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (121 ms) 2024-03-07T03:49:42.409 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTryAcquireLock 2024-03-07T03:49:42.409 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:42.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:49:42 smithi195 bash[59134]: ts=2024-03-07T03:49:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:43.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:42 smithi148 bash[79819]: cluster 2024-03-07T03:49:41.935383+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2024-03-07T03:49:43.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:42 smithi148 bash[79819]: audit 2024-03-07T03:49:42.352816+0000 mon.a (mon.0) 1881 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:43.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:42 smithi148 bash[79819]: cluster 2024-03-07T03:49:42.784598+0000 mon.a (mon.0) 1882 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2024-03-07T03:49:43.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:42 smithi148 bash[82222]: cluster 2024-03-07T03:49:41.935383+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2024-03-07T03:49:43.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:42 smithi148 bash[82222]: audit 2024-03-07T03:49:42.352816+0000 mon.a (mon.0) 1881 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:43.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:42 smithi148 bash[82222]: cluster 2024-03-07T03:49:42.784598+0000 mon.a (mon.0) 1882 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2024-03-07T03:49:43.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:42 smithi195 bash[66542]: cluster 2024-03-07T03:49:41.935383+0000 mon.a (mon.0) 1880 : cluster 0 osdmap e1041: 8 total, 8 up, 8 in 2024-03-07T03:49:43.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:42 smithi195 bash[66542]: audit 2024-03-07T03:49:42.352816+0000 mon.a (mon.0) 1881 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:43.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:42 smithi195 bash[66542]: cluster 2024-03-07T03:49:42.784598+0000 mon.a (mon.0) 1882 : cluster 0 osdmap e1042: 8 total, 8 up, 8 in 2024-03-07T03:49:44.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:43 smithi148 bash[79819]: cluster 2024-03-07T03:49:43.246500+0000 mgr.y (mgr.34101) 1073 : cluster 0 pgmap v1734: 105 pgs: 105 active+clean; 494 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.2 MiB/s wr, 303 op/s 2024-03-07T03:49:44.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:43 smithi148 bash[82222]: cluster 2024-03-07T03:49:43.246500+0000 mgr.y (mgr.34101) 1073 : cluster 0 pgmap v1734: 105 pgs: 105 active+clean; 494 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.2 MiB/s wr, 303 op/s 2024-03-07T03:49:44.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:43 smithi195 bash[66542]: cluster 2024-03-07T03:49:43.246500+0000 mgr.y (mgr.34101) 1073 : cluster 0 pgmap v1734: 105 pgs: 105 active+clean; 494 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 8.2 MiB/s wr, 303 op/s 2024-03-07T03:49:46.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:46 smithi195 bash[66542]: cluster 2024-03-07T03:49:45.247468+0000 mgr.y (mgr.34101) 1074 : cluster 0 pgmap v1735: 105 pgs: 105 active+clean; 532 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 10 MiB/s wr, 276 op/s 2024-03-07T03:49:46.800 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:46 smithi148 bash[79819]: cluster 2024-03-07T03:49:45.247468+0000 mgr.y (mgr.34101) 1074 : cluster 0 pgmap v1735: 105 pgs: 105 active+clean; 532 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 10 MiB/s wr, 276 op/s 2024-03-07T03:49:46.800 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:46 smithi148 bash[82222]: cluster 2024-03-07T03:49:45.247468+0000 mgr.y (mgr.34101) 1074 : cluster 0 pgmap v1735: 105 pgs: 105 active+clean; 532 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.9 MiB/s rd, 10 MiB/s wr, 276 op/s 2024-03-07T03:49:47.504 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/0.DiffIterateTryAcquireLock (5095 ms) 2024-03-07T03:49:47.504 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-03-07T03:49:47.504 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_librbd.cc:8289: Skipped 2024-03-07T03:49:47.505 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:49:47.505 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-03-07T03:49:47.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:49:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:49:47] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-03-07T03:49:47.789 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 13 tests from DiffIterateTest/0 (63904 ms total) 2024-03-07T03:49:47.789 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:49:47.789 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 13 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-03-07T03:49:47.840 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-03-07T03:49:49.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:48 smithi148 bash[79819]: cluster 2024-03-07T03:49:47.248476+0000 mgr.y (mgr.34101) 1075 : cluster 0 pgmap v1736: 105 pgs: 105 active+clean; 546 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.5 MiB/s wr, 238 op/s 2024-03-07T03:49:49.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:48 smithi148 bash[79819]: cluster 2024-03-07T03:49:47.786507+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2024-03-07T03:49:49.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:48 smithi148 bash[82222]: cluster 2024-03-07T03:49:47.248476+0000 mgr.y (mgr.34101) 1075 : cluster 0 pgmap v1736: 105 pgs: 105 active+clean; 546 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.5 MiB/s wr, 238 op/s 2024-03-07T03:49:49.050 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:48 smithi148 bash[82222]: cluster 2024-03-07T03:49:47.786507+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2024-03-07T03:49:49.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:48 smithi195 bash[66542]: cluster 2024-03-07T03:49:47.248476+0000 mgr.y (mgr.34101) 1075 : cluster 0 pgmap v1736: 105 pgs: 105 active+clean; 546 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.5 MiB/s wr, 238 op/s 2024-03-07T03:49:49.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:48 smithi195 bash[66542]: cluster 2024-03-07T03:49:47.786507+0000 mon.a (mon.0) 1883 : cluster 0 osdmap e1043: 8 total, 8 up, 8 in 2024-03-07T03:49:49.817 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:50.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:49 smithi195 bash[66542]: cluster 2024-03-07T03:49:48.801769+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2024-03-07T03:49:50.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:49 smithi195 bash[66542]: audit 2024-03-07T03:49:48.816808+0000 mon.a (mon.0) 1885 : audit 1 from='client.? 172.21.15.148:0/2613926887' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-45","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:49:50.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:49 smithi148 bash[79819]: cluster 2024-03-07T03:49:48.801769+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2024-03-07T03:49:50.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:49 smithi148 bash[79819]: audit 2024-03-07T03:49:48.816808+0000 mon.a (mon.0) 1885 : audit 1 from='client.? 172.21.15.148:0/2613926887' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-45","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:49:50.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:49 smithi148 bash[82222]: cluster 2024-03-07T03:49:48.801769+0000 mon.a (mon.0) 1884 : cluster 0 osdmap e1044: 8 total, 8 up, 8 in 2024-03-07T03:49:50.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:49 smithi148 bash[82222]: audit 2024-03-07T03:49:48.816808+0000 mon.a (mon.0) 1885 : audit 1 from='client.? 172.21.15.148:0/2613926887' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-45","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-03-07T03:49:51.071 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:50 smithi195 bash[66542]: cluster 2024-03-07T03:49:49.249325+0000 mgr.y (mgr.34101) 1076 : cluster 0 pgmap v1739: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:51.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:50 smithi195 bash[66542]: cluster 2024-03-07T03:49:49.795094+0000 mon.a (mon.0) 1886 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:49:51.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:50 smithi195 bash[66542]: audit 2024-03-07T03:49:49.804845+0000 mon.a (mon.0) 1887 : audit 1 from='client.? 172.21.15.148:0/2613926887' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:49:51.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:50 smithi195 bash[66542]: cluster 2024-03-07T03:49:49.814163+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2024-03-07T03:49:51.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[79819]: cluster 2024-03-07T03:49:49.249325+0000 mgr.y (mgr.34101) 1076 : cluster 0 pgmap v1739: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:51.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[79819]: cluster 2024-03-07T03:49:49.795094+0000 mon.a (mon.0) 1886 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:49:51.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[79819]: audit 2024-03-07T03:49:49.804845+0000 mon.a (mon.0) 1887 : audit 1 from='client.? 172.21.15.148:0/2613926887' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:49:51.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[79819]: cluster 2024-03-07T03:49:49.814163+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2024-03-07T03:49:51.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[82222]: cluster 2024-03-07T03:49:49.249325+0000 mgr.y (mgr.34101) 1076 : cluster 0 pgmap v1739: 105 pgs: 3 creating+peering, 29 unknown, 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:51.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[82222]: cluster 2024-03-07T03:49:49.795094+0000 mon.a (mon.0) 1886 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-07T03:49:51.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[82222]: audit 2024-03-07T03:49:49.804845+0000 mon.a (mon.0) 1887 : audit 1 from='client.? 172.21.15.148:0/2613926887' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi148-68562-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-03-07T03:49:51.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:50 smithi148 bash[82222]: cluster 2024-03-07T03:49:49.814163+0000 mon.a (mon.0) 1888 : cluster 0 osdmap e1045: 8 total, 8 up, 8 in 2024-03-07T03:49:51.892 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [4363305~98639,5133881~16841,5817382~64709,6848563~98653,8466555~83151,9796611~58432,10277691~22869,12078376~96771,13229860~88449,18690890~30477] 2024-03-07T03:49:52.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:51 smithi148 bash[79819]: cluster 2024-03-07T03:49:50.813253+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2024-03-07T03:49:52.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:51 smithi148 bash[82222]: cluster 2024-03-07T03:49:50.813253+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2024-03-07T03:49:52.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:51 smithi195 bash[66542]: cluster 2024-03-07T03:49:50.813253+0000 mon.a (mon.0) 1889 : cluster 0 osdmap e1046: 8 total, 8 up, 8 in 2024-03-07T03:49:52.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:49:52 smithi195 bash[59134]: ts=2024-03-07T03:49:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:49:52.867 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [0~16777216] 2024-03-07T03:49:52.887 INFO:tasks.workunit.client.0.smithi148.stdout: diff was [0~16777216] 2024-03-07T03:49:52.896 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5056 ms) 2024-03-07T03:49:52.896 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministic 2024-03-07T03:49:52.897 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:53.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:52 smithi148 bash[79819]: cluster 2024-03-07T03:49:51.250044+0000 mgr.y (mgr.34101) 1077 : cluster 0 pgmap v1742: 105 pgs: 3 creating+peering, 23 unknown, 79 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:53.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:52 smithi148 bash[79819]: cluster 2024-03-07T03:49:51.817779+0000 mon.a (mon.0) 1890 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:49:53.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:52 smithi148 bash[79819]: cluster 2024-03-07T03:49:51.834400+0000 mon.a (mon.0) 1891 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2024-03-07T03:49:53.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:52 smithi148 bash[82222]: cluster 2024-03-07T03:49:51.250044+0000 mgr.y (mgr.34101) 1077 : cluster 0 pgmap v1742: 105 pgs: 3 creating+peering, 23 unknown, 79 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:53.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:52 smithi148 bash[82222]: cluster 2024-03-07T03:49:51.817779+0000 mon.a (mon.0) 1890 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:49:53.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:52 smithi148 bash[82222]: cluster 2024-03-07T03:49:51.834400+0000 mon.a (mon.0) 1891 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2024-03-07T03:49:53.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:52 smithi195 bash[66542]: cluster 2024-03-07T03:49:51.250044+0000 mgr.y (mgr.34101) 1077 : cluster 0 pgmap v1742: 105 pgs: 3 creating+peering, 23 unknown, 79 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:49:53.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:52 smithi195 bash[66542]: cluster 2024-03-07T03:49:51.817779+0000 mon.a (mon.0) 1890 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-07T03:49:53.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:52 smithi195 bash[66542]: cluster 2024-03-07T03:49:51.834400+0000 mon.a (mon.0) 1891 : cluster 0 osdmap e1047: 8 total, 8 up, 8 in 2024-03-07T03:49:54.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:53 smithi148 bash[79819]: cluster 2024-03-07T03:49:52.845017+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2024-03-07T03:49:54.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:53 smithi148 bash[82222]: cluster 2024-03-07T03:49:52.845017+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2024-03-07T03:49:54.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:53 smithi195 bash[66542]: cluster 2024-03-07T03:49:52.845017+0000 mon.a (mon.0) 1892 : cluster 0 osdmap e1048: 8 total, 8 up, 8 in 2024-03-07T03:49:55.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:54 smithi148 bash[79819]: cluster 2024-03-07T03:49:53.250950+0000 mgr.y (mgr.34101) 1078 : cluster 0 pgmap v1745: 105 pgs: 3 creating+peering, 5 unknown, 97 active+clean; 4.2 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 86 KiB/s wr, 21 op/s 2024-03-07T03:49:55.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:54 smithi148 bash[79819]: cluster 2024-03-07T03:49:53.855777+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2024-03-07T03:49:55.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:54 smithi148 bash[82222]: cluster 2024-03-07T03:49:53.250950+0000 mgr.y (mgr.34101) 1078 : cluster 0 pgmap v1745: 105 pgs: 3 creating+peering, 5 unknown, 97 active+clean; 4.2 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 86 KiB/s wr, 21 op/s 2024-03-07T03:49:55.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:54 smithi148 bash[82222]: cluster 2024-03-07T03:49:53.855777+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2024-03-07T03:49:55.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:54 smithi195 bash[66542]: cluster 2024-03-07T03:49:53.250950+0000 mgr.y (mgr.34101) 1078 : cluster 0 pgmap v1745: 105 pgs: 3 creating+peering, 5 unknown, 97 active+clean; 4.2 MiB data, 645 MiB used, 715 GiB / 715 GiB avail; 12 KiB/s rd, 86 KiB/s wr, 21 op/s 2024-03-07T03:49:55.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:54 smithi195 bash[66542]: cluster 2024-03-07T03:49:53.855777+0000 mon.a (mon.0) 1893 : cluster 0 osdmap e1049: 8 total, 8 up, 8 in 2024-03-07T03:49:55.939 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministic (3043 ms) 2024-03-07T03:49:55.939 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicPP 2024-03-07T03:49:55.939 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:56.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:55 smithi148 bash[79819]: cluster 2024-03-07T03:49:54.865897+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2024-03-07T03:49:56.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:55 smithi148 bash[82222]: cluster 2024-03-07T03:49:54.865897+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2024-03-07T03:49:56.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:55 smithi195 bash[66542]: cluster 2024-03-07T03:49:54.865897+0000 mon.a (mon.0) 1894 : cluster 0 osdmap e1050: 8 total, 8 up, 8 in 2024-03-07T03:49:57.176 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:56 smithi148 bash[82222]: cluster 2024-03-07T03:49:55.251677+0000 mgr.y (mgr.34101) 1079 : cluster 0 pgmap v1748: 105 pgs: 105 active+clean; 9.9 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 246 KiB/s wr, 27 op/s 2024-03-07T03:49:57.176 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:56 smithi148 bash[82222]: cluster 2024-03-07T03:49:55.879897+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2024-03-07T03:49:57.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:56 smithi148 bash[79819]: cluster 2024-03-07T03:49:55.251677+0000 mgr.y (mgr.34101) 1079 : cluster 0 pgmap v1748: 105 pgs: 105 active+clean; 9.9 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 246 KiB/s wr, 27 op/s 2024-03-07T03:49:57.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:56 smithi148 bash[79819]: cluster 2024-03-07T03:49:55.879897+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2024-03-07T03:49:57.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:56 smithi195 bash[66542]: cluster 2024-03-07T03:49:55.251677+0000 mgr.y (mgr.34101) 1079 : cluster 0 pgmap v1748: 105 pgs: 105 active+clean; 9.9 MiB data, 647 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 246 KiB/s wr, 27 op/s 2024-03-07T03:49:57.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:56 smithi195 bash[66542]: cluster 2024-03-07T03:49:55.879897+0000 mon.a (mon.0) 1895 : cluster 0 osdmap e1051: 8 total, 8 up, 8 in 2024-03-07T03:49:57.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:49:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:49:57] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.43.0" 2024-03-07T03:49:58.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:57 smithi148 bash[79819]: cluster 2024-03-07T03:49:56.889900+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2024-03-07T03:49:58.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:57 smithi148 bash[79819]: audit 2024-03-07T03:49:57.352618+0000 mon.a (mon.0) 1897 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:58.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:57 smithi148 bash[82222]: cluster 2024-03-07T03:49:56.889900+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2024-03-07T03:49:58.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:57 smithi148 bash[82222]: audit 2024-03-07T03:49:57.352618+0000 mon.a (mon.0) 1897 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:58.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:57 smithi195 bash[66542]: cluster 2024-03-07T03:49:56.889900+0000 mon.a (mon.0) 1896 : cluster 0 osdmap e1052: 8 total, 8 up, 8 in 2024-03-07T03:49:58.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:57 smithi195 bash[66542]: audit 2024-03-07T03:49:57.352618+0000 mon.a (mon.0) 1897 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:49:58.969 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicPP (3030 ms) 2024-03-07T03:49:58.970 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-03-07T03:49:58.970 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:49:59.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:58 smithi148 bash[79819]: cluster 2024-03-07T03:49:57.252408+0000 mgr.y (mgr.34101) 1080 : cluster 0 pgmap v1751: 105 pgs: 105 active+clean; 16 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 262 KiB/s wr, 78 op/s 2024-03-07T03:49:59.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:58 smithi148 bash[79819]: cluster 2024-03-07T03:49:57.903404+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2024-03-07T03:49:59.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:58 smithi148 bash[82222]: cluster 2024-03-07T03:49:57.252408+0000 mgr.y (mgr.34101) 1080 : cluster 0 pgmap v1751: 105 pgs: 105 active+clean; 16 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 262 KiB/s wr, 78 op/s 2024-03-07T03:49:59.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:58 smithi148 bash[82222]: cluster 2024-03-07T03:49:57.903404+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2024-03-07T03:49:59.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:58 smithi195 bash[66542]: cluster 2024-03-07T03:49:57.252408+0000 mgr.y (mgr.34101) 1080 : cluster 0 pgmap v1751: 105 pgs: 105 active+clean; 16 MiB data, 649 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 262 KiB/s wr, 78 op/s 2024-03-07T03:49:59.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:58 smithi195 bash[66542]: cluster 2024-03-07T03:49:57.903404+0000 mon.a (mon.0) 1898 : cluster 0 osdmap e1053: 8 total, 8 up, 8 in 2024-03-07T03:50:00.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:59 smithi148 bash[82222]: cluster 2024-03-07T03:49:58.901360+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2024-03-07T03:50:00.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:49:59 smithi148 bash[82222]: cluster 2024-03-07T03:49:59.253182+0000 mgr.y (mgr.34101) 1081 : cluster 0 pgmap v1754: 105 pgs: 105 active+clean; 16 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 104 KiB/s wr, 84 op/s 2024-03-07T03:50:00.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:59 smithi148 bash[79819]: cluster 2024-03-07T03:49:58.901360+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2024-03-07T03:50:00.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:49:59 smithi148 bash[79819]: cluster 2024-03-07T03:49:59.253182+0000 mgr.y (mgr.34101) 1081 : cluster 0 pgmap v1754: 105 pgs: 105 active+clean; 16 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 104 KiB/s wr, 84 op/s 2024-03-07T03:50:00.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:00 smithi148 bash[79819]: debug 2024-03-07T03:49:59.993+0000 7f1956045700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:00.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:59 smithi195 bash[66542]: cluster 2024-03-07T03:49:58.901360+0000 mon.a (mon.0) 1899 : cluster 0 osdmap e1054: 8 total, 8 up, 8 in 2024-03-07T03:50:00.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:49:59 smithi195 bash[66542]: cluster 2024-03-07T03:49:59.253182+0000 mgr.y (mgr.34101) 1081 : cluster 0 pgmap v1754: 105 pgs: 105 active+clean; 16 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 104 KiB/s wr, 84 op/s 2024-03-07T03:50:01.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:00 smithi148 bash[79819]: cluster 2024-03-07T03:49:59.925338+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2024-03-07T03:50:01.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:00 smithi148 bash[79819]: cluster 2024-03-07T03:50:00.000176+0000 mon.a (mon.0) 1901 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:01.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:00 smithi148 bash[82222]: cluster 2024-03-07T03:49:59.925338+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2024-03-07T03:50:01.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:00 smithi148 bash[82222]: cluster 2024-03-07T03:50:00.000176+0000 mon.a (mon.0) 1901 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:01.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:00 smithi195 bash[66542]: cluster 2024-03-07T03:49:59.925338+0000 mon.a (mon.0) 1900 : cluster 0 osdmap e1055: 8 total, 8 up, 8 in 2024-03-07T03:50:01.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:00 smithi195 bash[66542]: cluster 2024-03-07T03:50:00.000176+0000 mon.a (mon.0) 1901 : cluster 4 overall HEALTH_ERR 8 osds(s) are not reachable 2024-03-07T03:50:02.005 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (3036 ms) 2024-03-07T03:50:02.005 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-03-07T03:50:02.006 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:02.144 INFO:tasks.workunit.client.0.smithi148.stdout: i=0 exists [12914769~2378166,36159587~8056706,85079519~2376903,145632002~711245,168027210~4403753,186946093~7822437,236884395~4555699,259568006~5523512,268903105~2278258,330112257~5085367] wrote [12914769~2378166,36159587~8056706,85079519~2376903,145632002~711245,168027210~4403753,186946093~7822437,236884395~4555699,259568006~5523512,268903105~2278258,330112257~5085367] 2024-03-07T03:50:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:01 smithi148 bash[79819]: cluster 2024-03-07T03:50:00.925474+0000 mon.a (mon.0) 1902 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2024-03-07T03:50:02.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:01 smithi148 bash[79819]: cluster 2024-03-07T03:50:01.253844+0000 mgr.y (mgr.34101) 1082 : cluster 0 pgmap v1757: 105 pgs: 105 active+clean; 16 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 9.0 KiB/s wr, 69 op/s 2024-03-07T03:50:02.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:01 smithi148 bash[82222]: cluster 2024-03-07T03:50:00.925474+0000 mon.a (mon.0) 1902 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2024-03-07T03:50:02.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:01 smithi148 bash[82222]: cluster 2024-03-07T03:50:01.253844+0000 mgr.y (mgr.34101) 1082 : cluster 0 pgmap v1757: 105 pgs: 105 active+clean; 16 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 9.0 KiB/s wr, 69 op/s 2024-03-07T03:50:02.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:01 smithi195 bash[66542]: cluster 2024-03-07T03:50:00.925474+0000 mon.a (mon.0) 1902 : cluster 0 osdmap e1056: 8 total, 8 up, 8 in 2024-03-07T03:50:02.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:01 smithi195 bash[66542]: cluster 2024-03-07T03:50:01.253844+0000 mgr.y (mgr.34101) 1082 : cluster 0 pgmap v1757: 105 pgs: 105 active+clean; 16 MiB data, 650 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 9.0 KiB/s wr, 69 op/s 2024-03-07T03:50:02.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:50:02 smithi195 bash[59134]: ts=2024-03-07T03:50:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:02.898 INFO:tasks.workunit.client.0.smithi148.stdout: i=1 exists [12914769~2378166,34549737~14843132,80298012~1975515,85079519~2376903,145632002~711245,168027210~4403753,186946093~7822437,205622652~2759711,236884395~4555699,259568006~5523512,268903105~2278258,304636079~8022992,323755567~11442057,356242395~926827] wrote [34549737~6172299,41940403~7452466,80298012~1975515,205622652~2759711,237670573~1368339,304636079~8022992,323755567~9152577,356242395~926827] 2024-03-07T03:50:03.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:02 smithi148 bash[79819]: cluster 2024-03-07T03:50:01.947287+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2024-03-07T03:50:03.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:02 smithi148 bash[79819]: cluster 2024-03-07T03:50:02.796105+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2024-03-07T03:50:03.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:02 smithi148 bash[82222]: cluster 2024-03-07T03:50:01.947287+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2024-03-07T03:50:03.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:02 smithi148 bash[82222]: cluster 2024-03-07T03:50:02.796105+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2024-03-07T03:50:03.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:02 smithi195 bash[66542]: cluster 2024-03-07T03:50:01.947287+0000 mon.a (mon.0) 1903 : cluster 0 osdmap e1057: 8 total, 8 up, 8 in 2024-03-07T03:50:03.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:02 smithi195 bash[66542]: cluster 2024-03-07T03:50:02.796105+0000 mon.a (mon.0) 1904 : cluster 0 osdmap e1058: 8 total, 8 up, 8 in 2024-03-07T03:50:04.030 INFO:tasks.workunit.client.0.smithi148.stdout: i=2 exists [12914769~2378166,30417302~845709,34549737~16771152,80298012~1975515,85079519~2376903,105422525~2858179,145632002~711245,148920064~5239011,168027210~4403753,173700043~7865129,186946093~7822437,205622652~2759711,236884395~4555699,259568006~5523512,268903105~2278258,272632846~1906509,304636079~8022992,316209242~1555698,323755567~11442057,345619743~3217256,356242395~926827,364372067~544595,405636826~827460] wrote [30417302~845709,45421247~5899642,105422525~2858179,148920064~5239011,173700043~7865129,272632846~1906509,316209242~1555698,345619743~3217256,364372067~544595,405636826~827460] 2024-03-07T03:50:04.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:03 smithi148 bash[82222]: cluster 2024-03-07T03:50:03.254760+0000 mgr.y (mgr.34101) 1083 : cluster 0 pgmap v1760: 105 pgs: 105 active+clean; 60 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 10 MiB/s wr, 115 op/s 2024-03-07T03:50:04.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:03 smithi148 bash[82222]: audit 2024-03-07T03:50:03.902563+0000 mon.a (mon.0) 1905 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:04.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:03 smithi148 bash[79819]: cluster 2024-03-07T03:50:03.254760+0000 mgr.y (mgr.34101) 1083 : cluster 0 pgmap v1760: 105 pgs: 105 active+clean; 60 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 10 MiB/s wr, 115 op/s 2024-03-07T03:50:04.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:03 smithi148 bash[79819]: audit 2024-03-07T03:50:03.902563+0000 mon.a (mon.0) 1905 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:04.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:03 smithi195 bash[66542]: cluster 2024-03-07T03:50:03.254760+0000 mgr.y (mgr.34101) 1083 : cluster 0 pgmap v1760: 105 pgs: 105 active+clean; 60 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 77 KiB/s rd, 10 MiB/s wr, 115 op/s 2024-03-07T03:50:04.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:03 smithi195 bash[66542]: audit 2024-03-07T03:50:03.902563+0000 mon.a (mon.0) 1905 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:05.096 INFO:tasks.workunit.client.0.smithi148.stdout: i=3 exists [411708~1774038,12914769~2378166,30417302~845709,34549737~16771152,80298012~1975515,85079519~2376903,105422525~2858179,113898140~3529385,145632002~711245,148920064~5239011,168027210~4403753,173700043~7865129,186946093~11052280,205622652~2759711,236884395~4555699,259568006~5523512,268903105~2278258,272632846~1906509,287804950~6807570,304636079~8022992,316209242~1555698,323755567~11793988,345619743~3217256,356242395~926827,364372067~544595,398262032~7235035,405636826~827460,406631340~7153738] wrote [411708~1774038,40928161~4682706,113898140~3529385,172026743~136272,192344590~5653783,287804950~6807570,327771314~7778241,398262032~7235035,406631340~7153738] 2024-03-07T03:50:05.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:04 smithi148 bash[79819]: cluster 2024-03-07T03:50:03.962555+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2024-03-07T03:50:05.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:04 smithi148 bash[82222]: cluster 2024-03-07T03:50:03.962555+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2024-03-07T03:50:05.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:04 smithi195 bash[66542]: cluster 2024-03-07T03:50:03.962555+0000 mon.a (mon.0) 1906 : cluster 0 osdmap e1059: 8 total, 8 up, 8 in 2024-03-07T03:50:06.116 INFO:tasks.workunit.client.0.smithi148.stdout: i=4 exists [411708~1774038,9613849~3050732,12853991~2642681,30417302~845709,34549737~16771152,80298012~1975515,85079519~2376903,100868395~7412309,113898140~3529385,145632002~711245,148920064~5239011,168027210~4403753,173700043~7865129,186946093~11052280,198728428~9653935,234623570~6816524,259568006~5523512,268903105~2278258,272632846~2829919,287804950~6807570,304636079~8022992,316209242~1555698,323755567~11793988,345619743~3217256,356242395~926827,364372067~544595,376874036~8148324,398262032~7235035,405636826~827460,406631340~7153738] wrote [9613849~3050732,12853991~2642681,38782170~5701072,100868395~7397440,198728428~7052438,234623570~4066972,274099027~1363738,292052119~2081505,376874036~8148324] 2024-03-07T03:50:06.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:05 smithi148 bash[79819]: cluster 2024-03-07T03:50:04.970074+0000 mon.a (mon.0) 1907 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2024-03-07T03:50:06.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:05 smithi148 bash[79819]: cluster 2024-03-07T03:50:05.255517+0000 mgr.y (mgr.34101) 1084 : cluster 0 pgmap v1763: 105 pgs: 105 active+clean; 120 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 20 MiB/s wr, 186 op/s 2024-03-07T03:50:06.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:05 smithi148 bash[82222]: cluster 2024-03-07T03:50:04.970074+0000 mon.a (mon.0) 1907 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2024-03-07T03:50:06.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:05 smithi148 bash[82222]: cluster 2024-03-07T03:50:05.255517+0000 mgr.y (mgr.34101) 1084 : cluster 0 pgmap v1763: 105 pgs: 105 active+clean; 120 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 20 MiB/s wr, 186 op/s 2024-03-07T03:50:06.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:05 smithi195 bash[66542]: cluster 2024-03-07T03:50:04.970074+0000 mon.a (mon.0) 1907 : cluster 0 osdmap e1060: 8 total, 8 up, 8 in 2024-03-07T03:50:06.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:05 smithi195 bash[66542]: cluster 2024-03-07T03:50:05.255517+0000 mgr.y (mgr.34101) 1084 : cluster 0 pgmap v1763: 105 pgs: 105 active+clean; 120 MiB data, 838 MiB used, 714 GiB / 715 GiB avail; 113 KiB/s rd, 20 MiB/s wr, 186 op/s 2024-03-07T03:50:07.028 INFO:tasks.workunit.client.0.smithi148.stdout: i=5 exists [411708~1774038,9613849~3050732,12853991~2642681,30417302~845709,34549737~16771152,60951312~1945210,80298012~7286888,100868395~7412309,113898140~3529385,117916857~372552,145632002~711245,148920064~5239011,168027210~4403753,173700043~7865129,186946093~11052280,198728428~9653935,234623570~6816524,259568006~5523512,268903105~2278258,272632846~2829919,287804950~6807570,304636079~8022992,316209242~1555698,323755567~11793988,340245220~301926,345619743~3217256,355743407~1425815,364372067~544595,365819857~2328207,373818329~11204031,396104400~1910971,398262032~7235035,405636826~827460,406631340~7153738] wrote [50651257~311968,60951312~1945210,80462732~7122168,117916857~372552,328586357~1736230,340245220~301926,355743407~594893,365819857~2328207,373818329~6987688,396104400~1910971] 2024-03-07T03:50:07.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:06 smithi148 bash[79819]: cluster 2024-03-07T03:50:05.984354+0000 mon.a (mon.0) 1908 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2024-03-07T03:50:07.300 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:50:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:50:07] "GET /metrics HTTP/1.1" 200 36724 "" "Prometheus/2.43.0" 2024-03-07T03:50:07.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:06 smithi148 bash[82222]: cluster 2024-03-07T03:50:05.984354+0000 mon.a (mon.0) 1908 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2024-03-07T03:50:07.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:06 smithi195 bash[66542]: cluster 2024-03-07T03:50:05.984354+0000 mon.a (mon.0) 1908 : cluster 0 osdmap e1061: 8 total, 8 up, 8 in 2024-03-07T03:50:08.102 INFO:tasks.workunit.client.0.smithi148.stdout: i=6 exists [411708~1774038,8236609~7260063,30417302~845709,34549737~16771152,60951312~1945210,80298012~10922566,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,141595659~2000098,145632002~711245,148920064~5239011,168027210~4403753,173700043~7865129,186946093~11052280,198728428~9653935,217342900~7631002,234623570~6816524,259568006~5523512,267085344~4096019,272632846~2829919,287804950~6807570,304636079~8022992,316209242~1555698,320256180~15293375,340245220~301926,345619743~3217256,355743407~1425815,364372067~544595,365819857~2328207,373818329~11204031,396104400~1910971,398262032~7235035,405636826~827460,406631340~7153738] wrote [8236609~6502121,30447786~199618,41242837~4180672,87105239~4115339,125848539~1054257,141595659~2000098,217342900~7631002,267085344~2776123,320256180~6899360] 2024-03-07T03:50:08.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:08 smithi148 bash[79819]: cluster 2024-03-07T03:50:06.986144+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2024-03-07T03:50:08.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:08 smithi148 bash[79819]: cluster 2024-03-07T03:50:07.256240+0000 mgr.y (mgr.34101) 1085 : cluster 0 pgmap v1766: 105 pgs: 105 active+clean; 154 MiB data, 947 MiB used, 714 GiB / 715 GiB avail; 116 KiB/s rd, 17 MiB/s wr, 186 op/s 2024-03-07T03:50:08.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:08 smithi148 bash[82222]: cluster 2024-03-07T03:50:06.986144+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2024-03-07T03:50:08.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:08 smithi148 bash[82222]: cluster 2024-03-07T03:50:07.256240+0000 mgr.y (mgr.34101) 1085 : cluster 0 pgmap v1766: 105 pgs: 105 active+clean; 154 MiB data, 947 MiB used, 714 GiB / 715 GiB avail; 116 KiB/s rd, 17 MiB/s wr, 186 op/s 2024-03-07T03:50:08.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:08 smithi195 bash[66542]: cluster 2024-03-07T03:50:06.986144+0000 mon.a (mon.0) 1909 : cluster 0 osdmap e1062: 8 total, 8 up, 8 in 2024-03-07T03:50:08.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:08 smithi195 bash[66542]: cluster 2024-03-07T03:50:07.256240+0000 mgr.y (mgr.34101) 1085 : cluster 0 pgmap v1766: 105 pgs: 105 active+clean; 154 MiB data, 947 MiB used, 714 GiB / 715 GiB avail; 116 KiB/s rd, 17 MiB/s wr, 186 op/s 2024-03-07T03:50:09.101 INFO:tasks.workunit.client.0.smithi148.stdout: i=7 exists [411708~1774038,8236609~7260063,30417302~845709,34549737~16771152,57803779~5092743,80298012~10922566,92549043~7670913,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,141595659~2000098,145632002~711245,148920064~5239011,165699741~6731222,173700043~7865129,186216281~11782092,198728428~9653935,212721067~4572246,217342900~7631002,234623570~6816524,259568006~5523512,267085344~4096019,272632846~7578132,287804950~6807570,304636079~8022992,316209242~1555698,320256180~15293375,340245220~301926,342653401~87233,345619743~3217256,355743407~1425815,364372067~544595,365819857~2328207,373818329~11204031,396104400~1910971,398262032~7235035,405636826~827460,406631340~7153738] wrote [57803779~4200837,92549043~7670913,115348902~179863,165699741~2638773,186216281~2814800,212721067~4572246,222082125~422332,274685500~5525478,342653401~87233,381503920~354095] 2024-03-07T03:50:09.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:09 smithi148 bash[79819]: cluster 2024-03-07T03:50:08.010287+0000 mon.a (mon.0) 1910 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2024-03-07T03:50:09.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:09 smithi148 bash[82222]: cluster 2024-03-07T03:50:08.010287+0000 mon.a (mon.0) 1910 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2024-03-07T03:50:09.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:09 smithi195 bash[66542]: cluster 2024-03-07T03:50:08.010287+0000 mon.a (mon.0) 1910 : cluster 0 osdmap e1063: 8 total, 8 up, 8 in 2024-03-07T03:50:10.139 INFO:tasks.workunit.client.0.smithi148.stdout: i=8 exists [411708~1774038,8236609~7260063,30417302~845709,34549737~21988250,57803779~5092743,73318339~17902239,92549043~7670913,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,141595659~2000098,145632002~711245,148920064~5239011,157765702~4120283,165699741~6731222,173700043~7865129,186216281~11782092,198723892~9658471,212721067~4572246,217342900~7631002,234623570~6816524,248790554~4704673,259568006~5523512,267085344~4096019,272632846~7578132,287804950~6807570,304636079~8022992,316209242~1555698,320256180~15293375,340245220~301926,342653401~87233,345619743~3217256,349125750~8043472,364372067~544595,365819857~4835469,373818329~11204031,396104400~1910971,398262032~15523046] wrote [50991332~5546655,73318339~7719461,82548005~2513048,157765702~4120283,198723892~7734567,248790554~4704673,349125750~6808036,367167674~3487652,403977114~5166446] 2024-03-07T03:50:10.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:10 smithi148 bash[79819]: cluster 2024-03-07T03:50:09.015175+0000 mon.a (mon.0) 1911 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2024-03-07T03:50:10.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:10 smithi148 bash[79819]: cluster 2024-03-07T03:50:09.257085+0000 mgr.y (mgr.34101) 1086 : cluster 0 pgmap v1769: 105 pgs: 105 active+clean; 303 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 35 MiB/s wr, 218 op/s 2024-03-07T03:50:10.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:10 smithi148 bash[82222]: cluster 2024-03-07T03:50:09.015175+0000 mon.a (mon.0) 1911 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2024-03-07T03:50:10.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:10 smithi148 bash[82222]: cluster 2024-03-07T03:50:09.257085+0000 mgr.y (mgr.34101) 1086 : cluster 0 pgmap v1769: 105 pgs: 105 active+clean; 303 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 35 MiB/s wr, 218 op/s 2024-03-07T03:50:10.321 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:50:10 smithi195 bash[49044]: logger=cleanup t=2024-03-07T03:50:10.180311528Z level=info msg="Completed cleanup jobs" duration=67.488181ms 2024-03-07T03:50:10.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:10 smithi195 bash[66542]: cluster 2024-03-07T03:50:09.015175+0000 mon.a (mon.0) 1911 : cluster 0 osdmap e1064: 8 total, 8 up, 8 in 2024-03-07T03:50:10.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:10 smithi195 bash[66542]: cluster 2024-03-07T03:50:09.257085+0000 mgr.y (mgr.34101) 1086 : cluster 0 pgmap v1769: 105 pgs: 105 active+clean; 303 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 35 MiB/s wr, 218 op/s 2024-03-07T03:50:11.194 INFO:tasks.workunit.client.0.smithi148.stdout: i=9 exists [411708~1774038,6532219~12529447,30417302~845709,34549737~21988250,57803779~5092743,73318339~26901617,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,141595659~2000098,145632002~711245,148920064~5239011,157765702~4120283,165699741~6731222,173700043~7865129,186216281~11782092,198723892~9658471,212721067~4572246,217342900~7631002,234623570~6816524,248790554~5228083,259568006~11613357,272632846~12330862,287804950~6807570,304636079~8022992,316209242~1555698,320256180~15293375,340245220~301926,342653401~87233,345619743~3217256,349125750~14089016,364372067~544595,365819857~4835469,373818329~11204031,396104400~1910971,398262032~15523046] wrote [6532219~12529447,51611524~4071078,75514590~7468469,90437201~6556471,175304430~4515577,251844503~2174134,262020042~7307099,279142998~5820710,355738104~7476662] 2024-03-07T03:50:11.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:11 smithi148 bash[79819]: cluster 2024-03-07T03:50:10.040877+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2024-03-07T03:50:11.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:11 smithi148 bash[82222]: cluster 2024-03-07T03:50:10.040877+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2024-03-07T03:50:11.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:11 smithi195 bash[66542]: cluster 2024-03-07T03:50:10.040877+0000 mon.a (mon.0) 1912 : cluster 0 osdmap e1065: 8 total, 8 up, 8 in 2024-03-07T03:50:12.172 INFO:tasks.workunit.client.0.smithi148.stdout: i=10 exists [411708~1774038,6532219~12529447,30417302~845709,34549737~21988250,57803779~5092743,73318339~26901617,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,141595659~2000098,145632002~711245,148920064~5239011,157765702~4120283,165699741~6731222,173700043~7865129,186216281~11782092,198723892~9658471,212721067~12252835,225821032~2159048,234623570~19395067,259568006~11613357,272632846~12330862,287804950~6807570,304636079~30913476,340245220~301926,342653401~87233,345619743~3217256,349125750~14089016,364372067~544595,365819857~4835469,373818329~11204031,387840253~857556,396104400~1910971,398262032~15523046] wrote [43215445~2565012,82377928~220449,217146314~3982069,225821032~2159048,237176469~11859862,310199625~11499582,387840253~857556,398363924~5283318] 2024-03-07T03:50:12.312 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:12 smithi195 bash[66542]: cluster 2024-03-07T03:50:11.041907+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2024-03-07T03:50:12.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:12 smithi195 bash[66542]: cluster 2024-03-07T03:50:11.257639+0000 mgr.y (mgr.34101) 1087 : cluster 0 pgmap v1772: 105 pgs: 105 active+clean; 346 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 39 MiB/s wr, 165 op/s 2024-03-07T03:50:12.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:12 smithi195 bash[66542]: audit 2024-03-07T03:50:11.924102+0000 mon.a (mon.0) 1914 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:50:12.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:12 smithi148 bash[79819]: cluster 2024-03-07T03:50:11.041907+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2024-03-07T03:50:12.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:12 smithi148 bash[79819]: cluster 2024-03-07T03:50:11.257639+0000 mgr.y (mgr.34101) 1087 : cluster 0 pgmap v1772: 105 pgs: 105 active+clean; 346 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 39 MiB/s wr, 165 op/s 2024-03-07T03:50:12.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:12 smithi148 bash[79819]: audit 2024-03-07T03:50:11.924102+0000 mon.a (mon.0) 1914 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:50:12.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:12 smithi148 bash[82222]: cluster 2024-03-07T03:50:11.041907+0000 mon.a (mon.0) 1913 : cluster 0 osdmap e1066: 8 total, 8 up, 8 in 2024-03-07T03:50:12.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:12 smithi148 bash[82222]: cluster 2024-03-07T03:50:11.257639+0000 mgr.y (mgr.34101) 1087 : cluster 0 pgmap v1772: 105 pgs: 105 active+clean; 346 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 39 MiB/s wr, 165 op/s 2024-03-07T03:50:12.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:12 smithi148 bash[82222]: audit 2024-03-07T03:50:11.924102+0000 mon.a (mon.0) 1914 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:50:12.571 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:50:12 smithi195 bash[59134]: ts=2024-03-07T03:50:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:13.180 INFO:tasks.workunit.client.0.smithi148.stdout: i=11 exists [411708~1774038,6532219~12529447,19457911~7445349,30417302~845709,34549737~21988250,57803779~5092743,73318339~26901617,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,141595659~2000098,145632002~711245,148920064~5239011,157765702~4120283,165699741~6731222,173700043~7865129,183777330~1611651,186216281~11782092,198723892~9658471,212721067~18244871,234623570~19395067,259568006~25395702,287804950~6807570,304636079~30913476,340245220~301926,342653401~87233,345619743~3217256,349125750~14089016,364372067~544595,365819857~4835469,373818329~11632532,387840253~857556,396104400~1910971,398262032~15523046] wrote [19457911~7445349,74533356~2167968,84985877~2925981,103776147~1508426,183777330~1611651,223445919~7520019,269407428~7126639,318769770~5157961,382358074~3092787] 2024-03-07T03:50:13.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:13 smithi195 bash[66542]: cluster 2024-03-07T03:50:12.066917+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2024-03-07T03:50:13.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:13 smithi195 bash[66542]: audit 2024-03-07T03:50:12.352983+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:13.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:13 smithi148 bash[79819]: cluster 2024-03-07T03:50:12.066917+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2024-03-07T03:50:13.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:13 smithi148 bash[79819]: audit 2024-03-07T03:50:12.352983+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:13.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:13 smithi148 bash[82222]: cluster 2024-03-07T03:50:12.066917+0000 mon.a (mon.0) 1915 : cluster 0 osdmap e1067: 8 total, 8 up, 8 in 2024-03-07T03:50:13.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:13 smithi148 bash[82222]: audit 2024-03-07T03:50:12.352983+0000 mon.a (mon.0) 1916 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:14.176 INFO:tasks.workunit.client.0.smithi148.stdout: i=12 exists [411708~1774038,6532219~12529447,19457911~7445349,30417302~845709,34549737~21988250,57803779~5092743,64325967~4601691,73318339~26901617,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,129702178~7842337,141595659~2000098,145632002~711245,148920064~12965921,165699741~6731222,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~19395067,256418721~28544987,287804950~6807570,304636079~30913476,340245220~301926,342653401~87233,345619743~3217256,349125750~14089016,364372067~544595,365819857~4835469,373818329~11632532,387840253~857556,396104400~1910971,398262032~15523046] wrote [59213040~1515451,64325967~4601691,129702178~7842337,150837857~7785023,175464469~2007937,183614465~1574978,256418721~4055993,291791504~1076160,380668072~3103693] 2024-03-07T03:50:14.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:14 smithi148 bash[82222]: cluster 2024-03-07T03:50:13.070197+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2024-03-07T03:50:14.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:14 smithi148 bash[82222]: cluster 2024-03-07T03:50:13.258571+0000 mgr.y (mgr.34101) 1088 : cluster 0 pgmap v1775: 105 pgs: 105 active+clean; 456 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 35 MiB/s wr, 109 op/s 2024-03-07T03:50:14.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:14 smithi148 bash[79819]: cluster 2024-03-07T03:50:13.070197+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2024-03-07T03:50:14.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:14 smithi148 bash[79819]: cluster 2024-03-07T03:50:13.258571+0000 mgr.y (mgr.34101) 1088 : cluster 0 pgmap v1775: 105 pgs: 105 active+clean; 456 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 35 MiB/s wr, 109 op/s 2024-03-07T03:50:14.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:14 smithi195 bash[66542]: cluster 2024-03-07T03:50:13.070197+0000 mon.a (mon.0) 1917 : cluster 0 osdmap e1068: 8 total, 8 up, 8 in 2024-03-07T03:50:14.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:14 smithi195 bash[66542]: cluster 2024-03-07T03:50:13.258571+0000 mgr.y (mgr.34101) 1088 : cluster 0 pgmap v1775: 105 pgs: 105 active+clean; 456 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 35 MiB/s wr, 109 op/s 2024-03-07T03:50:15.167 INFO:tasks.workunit.client.0.smithi148.stdout: i=13 exists [411708~1774038,6532219~12529447,19457911~7445349,30417302~845709,34549737~21988250,57803779~5092743,63303944~201367,64325967~8191214,73318339~26901617,100868395~7412309,113898140~3529385,117916857~372552,125848539~1054257,129702178~7842337,141595659~2000098,145632002~711245,148920064~12965921,162263462~10167501,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~21470474,256418721~28544987,287804950~6807570,304636079~30913476,340245220~301926,342653401~87233,345619743~3217256,349125750~14089016,364372067~544595,365819857~4835469,373818329~11632532,387840253~857556,396104400~1910971,398262032~15523046] wrote [63303944~201367,68255436~4261745,99098812~948897,162263462~4880819,222918706~2107243,227626693~362811,242952748~7092775,252651619~3442425,327459444~5624143] 2024-03-07T03:50:15.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:15 smithi148 bash[79819]: cluster 2024-03-07T03:50:14.085712+0000 mon.a (mon.0) 1918 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2024-03-07T03:50:15.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:15 smithi148 bash[82222]: cluster 2024-03-07T03:50:14.085712+0000 mon.a (mon.0) 1918 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2024-03-07T03:50:15.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:15 smithi195 bash[66542]: cluster 2024-03-07T03:50:14.085712+0000 mon.a (mon.0) 1918 : cluster 0 osdmap e1069: 8 total, 8 up, 8 in 2024-03-07T03:50:16.226 INFO:tasks.workunit.client.0.smithi148.stdout: i=14 exists [411708~1774038,6532219~20371041,30417302~845709,34549737~21988250,57803779~5092743,63303944~201367,64325967~8191214,73318339~26901617,100868395~7412309,113898140~3529385,117916857~372552,124548520~12995995,141595659~2000098,145632002~711245,148920064~12965921,162263462~10167501,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~21470474,256418721~28544987,287804950~6807570,304636079~30913476,338596182~6044549,345619743~3217256,349125750~14089016,364372067~544595,365819857~6651634,373818329~11632532,387840253~857556,396104400~17680678] wrote [14647790~5557880,75756321~5265413,124548520~7760363,214297366~4832134,247885410~2272686,261049193~7738338,319730223~3257281,338596182~6044549,366367017~6104474,396504991~4834417] 2024-03-07T03:50:16.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:16 smithi148 bash[79819]: cluster 2024-03-07T03:50:15.082245+0000 mon.a (mon.0) 1919 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2024-03-07T03:50:16.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:16 smithi148 bash[79819]: cluster 2024-03-07T03:50:15.259344+0000 mgr.y (mgr.34101) 1089 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 554 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 188 KiB/s rd, 50 MiB/s wr, 307 op/s 2024-03-07T03:50:16.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:16 smithi148 bash[82222]: cluster 2024-03-07T03:50:15.082245+0000 mon.a (mon.0) 1919 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2024-03-07T03:50:16.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:16 smithi148 bash[82222]: cluster 2024-03-07T03:50:15.259344+0000 mgr.y (mgr.34101) 1089 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 554 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 188 KiB/s rd, 50 MiB/s wr, 307 op/s 2024-03-07T03:50:16.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:16 smithi195 bash[66542]: cluster 2024-03-07T03:50:15.082245+0000 mon.a (mon.0) 1919 : cluster 0 osdmap e1070: 8 total, 8 up, 8 in 2024-03-07T03:50:16.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:16 smithi195 bash[66542]: cluster 2024-03-07T03:50:15.259344+0000 mgr.y (mgr.34101) 1089 : cluster 0 pgmap v1778: 105 pgs: 105 active+clean; 554 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 188 KiB/s rd, 50 MiB/s wr, 307 op/s 2024-03-07T03:50:17.206 INFO:tasks.workunit.client.0.smithi148.stdout: i=15 exists [411708~1774038,6532219~20371041,30417302~845709,34549737~21988250,57803779~5092743,63303944~201367,64325967~8191214,73318339~26901617,100868395~7412309,113898140~3529385,117916857~3467851,122765211~14779304,141595659~2000098,145632002~711245,148920064~12965921,162263462~10167501,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~21470474,256418721~38193799,304636079~30913476,338596182~10240817,349125750~14089016,364372067~544595,365819857~19631004,387840253~857556,396104400~17680678] wrote [13254854~2979971,23227576~888366,79469700~7652844,118019294~3365414,122765211~5692190,134604585~2120645,283224464~6262025,340320751~6452507,370397859~4260757] 2024-03-07T03:50:17.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:17 smithi148 bash[79819]: cluster 2024-03-07T03:50:16.104125+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2024-03-07T03:50:17.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:50:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:50:17] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:50:17.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:17 smithi148 bash[82222]: cluster 2024-03-07T03:50:16.104125+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2024-03-07T03:50:17.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:17 smithi195 bash[66542]: cluster 2024-03-07T03:50:16.104125+0000 mon.a (mon.0) 1920 : cluster 0 osdmap e1071: 8 total, 8 up, 8 in 2024-03-07T03:50:18.220 INFO:tasks.workunit.client.0.smithi148.stdout: i=16 exists [411708~1774038,6532219~20371041,30417302~845709,34549737~21988250,57803779~5092743,63303944~201367,64325967~8191214,73318339~26901617,100868395~7412309,113898140~3529385,117916857~3467851,122765211~17457032,140981252~5361995,148920064~12965921,162263462~10167501,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~21470474,256418721~38655292,304636079~30913476,336069483~12767516,349125750~14089016,364372067~544595,365819857~19631004,387840253~857556,393189192~20595886] wrote [136257628~3964615,140981252~4918515,253016875~2704808,272874409~7478477,294050027~1023986,327521737~3826601,336069483~4060483,367382882~3967219,393189192~7115742] 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[79819]: cluster 2024-03-07T03:50:17.107192+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[79819]: cluster 2024-03-07T03:50:17.259774+0000 mgr.y (mgr.34101) 1090 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 573 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 221 KiB/s rd, 30 MiB/s wr, 340 op/s 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[79819]: audit 2024-03-07T03:50:17.818681+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[79819]: audit 2024-03-07T03:50:17.827780+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[79819]: audit 2024-03-07T03:50:18.022090+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[79819]: audit 2024-03-07T03:50:18.034077+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[82222]: cluster 2024-03-07T03:50:17.107192+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[82222]: cluster 2024-03-07T03:50:17.259774+0000 mgr.y (mgr.34101) 1090 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 573 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 221 KiB/s rd, 30 MiB/s wr, 340 op/s 2024-03-07T03:50:18.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[82222]: audit 2024-03-07T03:50:17.818681+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[82222]: audit 2024-03-07T03:50:17.827780+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[82222]: audit 2024-03-07T03:50:18.022090+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:18 smithi148 bash[82222]: audit 2024-03-07T03:50:18.034077+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:18 smithi195 bash[66542]: cluster 2024-03-07T03:50:17.107192+0000 mon.a (mon.0) 1921 : cluster 0 osdmap e1072: 8 total, 8 up, 8 in 2024-03-07T03:50:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:18 smithi195 bash[66542]: cluster 2024-03-07T03:50:17.259774+0000 mgr.y (mgr.34101) 1090 : cluster 0 pgmap v1781: 105 pgs: 105 active+clean; 573 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 221 KiB/s rd, 30 MiB/s wr, 340 op/s 2024-03-07T03:50:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:18 smithi195 bash[66542]: audit 2024-03-07T03:50:17.818681+0000 mon.a (mon.0) 1922 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:18 smithi195 bash[66542]: audit 2024-03-07T03:50:17.827780+0000 mon.a (mon.0) 1923 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:18 smithi195 bash[66542]: audit 2024-03-07T03:50:18.022090+0000 mon.a (mon.0) 1924 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:18.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:18 smithi195 bash[66542]: audit 2024-03-07T03:50:18.034077+0000 mon.a (mon.0) 1925 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:19.226 INFO:tasks.workunit.client.0.smithi148.stdout: i=17 exists [411708~1774038,6158081~21488235,30417302~845709,34549737~21988250,57803779~5092743,63303944~201367,64325967~8191214,73318339~26901617,100868395~7412309,109294519~8133006,117916857~3467851,122765211~17457032,140981252~5361995,148920064~12965921,162263462~10167501,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~21470474,256418721~38655292,304636079~30913476,336069483~12767516,349125750~14089016,364372067~544595,365819857~20342295,387840253~857556,393189192~20595886] wrote [6158081~1549191,24648710~2997606,85860546~2714107,109294519~7117755,154742302~5503692,201359980~4157286,330611803~903770,375376471~10785681] 2024-03-07T03:50:19.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[79819]: cluster 2024-03-07T03:50:18.125882+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2024-03-07T03:50:19.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[79819]: audit 2024-03-07T03:50:18.663627+0000 mon.a (mon.0) 1927 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:50:19.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[79819]: audit 2024-03-07T03:50:18.665203+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:50:19.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[79819]: audit 2024-03-07T03:50:18.675101+0000 mon.a (mon.0) 1929 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:19.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[82222]: cluster 2024-03-07T03:50:18.125882+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2024-03-07T03:50:19.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[82222]: audit 2024-03-07T03:50:18.663627+0000 mon.a (mon.0) 1927 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:50:19.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[82222]: audit 2024-03-07T03:50:18.665203+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:50:19.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:19 smithi148 bash[82222]: audit 2024-03-07T03:50:18.675101+0000 mon.a (mon.0) 1929 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:19.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:19 smithi195 bash[66542]: cluster 2024-03-07T03:50:18.125882+0000 mon.a (mon.0) 1926 : cluster 0 osdmap e1073: 8 total, 8 up, 8 in 2024-03-07T03:50:19.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:19 smithi195 bash[66542]: audit 2024-03-07T03:50:18.663627+0000 mon.a (mon.0) 1927 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:50:19.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:19 smithi195 bash[66542]: audit 2024-03-07T03:50:18.665203+0000 mon.a (mon.0) 1928 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:50:19.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:19 smithi195 bash[66542]: audit 2024-03-07T03:50:18.675101+0000 mon.a (mon.0) 1929 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:50:20.249 INFO:tasks.workunit.client.0.smithi148.stdout: i=18 exists [411708~1774038,6158081~21488235,30417302~845709,34549737~21988250,57803779~5092743,63303944~201367,64325967~35893989,100868395~7412309,109294519~12090189,122765211~17457032,140981252~5361995,148920064~12965921,162263462~10167501,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~60450443,304636079~30913476,336069483~12767516,349125750~14089016,364372067~544595,365819857~20342295,387840253~857556,388933558~24851520] wrote [7773699~2556196,20381108~579815,46184582~532289,70203694~5324926,116571287~3133237,254135847~3634931,262464511~6933972,269459731~4622679,380404765~4185438,388933558~4272177] 2024-03-07T03:50:20.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:20 smithi148 bash[79819]: cluster 2024-03-07T03:50:19.135487+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2024-03-07T03:50:20.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:20 smithi148 bash[79819]: cluster 2024-03-07T03:50:19.260301+0000 mgr.y (mgr.34101) 1091 : cluster 0 pgmap v1784: 105 pgs: 105 active+clean; 708 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 36 MiB/s wr, 331 op/s 2024-03-07T03:50:20.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:20 smithi148 bash[82222]: cluster 2024-03-07T03:50:19.135487+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2024-03-07T03:50:20.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:20 smithi148 bash[82222]: cluster 2024-03-07T03:50:19.260301+0000 mgr.y (mgr.34101) 1091 : cluster 0 pgmap v1784: 105 pgs: 105 active+clean; 708 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 36 MiB/s wr, 331 op/s 2024-03-07T03:50:20.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:20 smithi195 bash[66542]: cluster 2024-03-07T03:50:19.135487+0000 mon.a (mon.0) 1930 : cluster 0 osdmap e1074: 8 total, 8 up, 8 in 2024-03-07T03:50:20.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:20 smithi195 bash[66542]: cluster 2024-03-07T03:50:19.260301+0000 mgr.y (mgr.34101) 1091 : cluster 0 pgmap v1784: 105 pgs: 105 active+clean; 708 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 214 KiB/s rd, 36 MiB/s wr, 331 op/s 2024-03-07T03:50:21.264 INFO:tasks.workunit.client.0.smithi148.stdout: i=19 exists [411708~1774038,6158081~21488235,30417302~845709,34549737~21988250,57803779~5092743,63303944~201367,64325967~35893989,100868395~7412309,109294519~12090189,122765211~17457032,140981252~5361995,148920064~12965921,162263462~10167501,173700043~7865129,183614465~1774516,186216281~11782092,198723892~9658471,212721067~18244871,234623570~60450443,304636079~30913476,336069483~12767516,349125750~14089016,364372067~544595,365819857~20342295,387840253~857556,388933558~24851520] wrote [42103313~4935905,49530147~5899220,95774711~851980,97972550~1853331,152337353~5182248,213551928~1231786,235307700~5702874,259093301~3254738,270159898~7296401,345269971~1405168] 2024-03-07T03:50:21.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:21 smithi148 bash[79819]: cluster 2024-03-07T03:50:20.146111+0000 mon.a (mon.0) 1931 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2024-03-07T03:50:21.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:21 smithi148 bash[82222]: cluster 2024-03-07T03:50:20.146111+0000 mon.a (mon.0) 1931 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2024-03-07T03:50:21.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:21 smithi195 bash[66542]: cluster 2024-03-07T03:50:20.146111+0000 mon.a (mon.0) 1931 : cluster 0 osdmap e1075: 8 total, 8 up, 8 in 2024-03-07T03:50:22.190 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 1 diff [33554432~12582912,79691776~4194304,205520896~4194304,234881024~4194304,301989888~8388608,322961408~12582912,352321536~4194304] 2024-03-07T03:50:22.191 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [33554432~12582912,79691776~4194304,205520896~4194304,234881024~4194304,301989888~8388608,322961408~12582912,352321536~4194304] 2024-03-07T03:50:22.213 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [33554432~16777216,79691776~4194304,205520896~4194304,234881024~4194304,301989888~12582912,322961408~12582912,352321536~8388608] 2024-03-07T03:50:22.213 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 2 diff [29360128~20971520,79691776~4194304,104857600~4194304,146800640~8388608,171966464~8388608,205520896~4194304,234881024~4194304,272629760~4194304,301989888~8388608,314572800~4194304,322961408~12582912,343932928~4194304,352321536~4194304,360710144~4194304,402653184~4194304] 2024-03-07T03:50:22.213 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [29360128~20971520,79691776~4194304,104857600~4194304,146800640~8388608,171966464~8388608,205520896~4194304,234881024~4194304,272629760~4194304,301989888~8388608,314572800~4194304,322961408~12582912,343932928~4194304,352321536~4194304,360710144~4194304,402653184~4194304] 2024-03-07T03:50:22.218 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [29360128~25165824,79691776~4194304,104857600~4194304,146800640~8388608,171966464~12582912,205520896~4194304,234881024~4194304,272629760~4194304,301989888~16777216,322961408~12582912,343932928~25165824,402653184~4194304] 2024-03-07T03:50:22.218 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 3 diff [0~4194304,29360128~20971520,79691776~4194304,104857600~4194304,113246208~4194304,146800640~8388608,171966464~8388608,188743680~8388608,205520896~4194304,234881024~4194304,272629760~4194304,285212672~8388608,301989888~8388608,314572800~4194304,322961408~12582912,343932928~4194304,352321536~4194304,360710144~4194304,394264576~16777216] 2024-03-07T03:50:22.218 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,29360128~20971520,79691776~4194304,104857600~4194304,113246208~4194304,146800640~8388608,171966464~8388608,188743680~8388608,205520896~4194304,234881024~4194304,272629760~4194304,285212672~8388608,301989888~8388608,314572800~4194304,322961408~12582912,343932928~4194304,352321536~4194304,360710144~4194304,394264576~16777216] 2024-03-07T03:50:22.223 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,29360128~25165824,79691776~4194304,104857600~4194304,113246208~4194304,146800640~8388608,171966464~12582912,188743680~12582912,205520896~4194304,234881024~4194304,272629760~4194304,285212672~12582912,301989888~16777216,322961408~16777216,343932928~25165824,394264576~20971520] 2024-03-07T03:50:22.224 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 4 diff [0~4194304,8388608~8388608,29360128~20971520,79691776~4194304,100663296~8388608,113246208~4194304,146800640~8388608,171966464~8388608,188743680~20971520,230686720~8388608,272629760~4194304,285212672~8388608,301989888~8388608,314572800~4194304,322961408~12582912,343932928~4194304,352321536~4194304,360710144~4194304,373293056~8388608,394264576~16777216] 2024-03-07T03:50:22.224 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,8388608~8388608,29360128~20971520,79691776~4194304,100663296~8388608,113246208~4194304,146800640~8388608,171966464~8388608,188743680~20971520,230686720~8388608,272629760~4194304,285212672~8388608,301989888~8388608,314572800~4194304,322961408~12582912,343932928~4194304,352321536~4194304,360710144~4194304,373293056~8388608,394264576~16777216] 2024-03-07T03:50:22.229 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,8388608~8388608,29360128~25165824,79691776~4194304,100663296~8388608,113246208~4194304,146800640~8388608,171966464~12582912,188743680~20971520,230686720~8388608,272629760~4194304,285212672~12582912,301989888~16777216,322961408~16777216,343932928~25165824,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.229 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 5 diff [0~4194304,8388608~8388608,29360128~20971520,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,146800640~8388608,171966464~8388608,188743680~20971520,230686720~8388608,272629760~4194304,285212672~8388608,301989888~8388608,314572800~4194304,322961408~12582912,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.229 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,8388608~8388608,29360128~20971520,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,146800640~8388608,171966464~8388608,188743680~20971520,230686720~8388608,272629760~4194304,285212672~8388608,301989888~8388608,314572800~4194304,322961408~12582912,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.238 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,8388608~8388608,29360128~25165824,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,146800640~8388608,171966464~12582912,188743680~20971520,230686720~8388608,272629760~4194304,285212672~12582912,301989888~16777216,322961408~46137344,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.239 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 6 diff [0~12582912,29360128~20971520,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,171966464~8388608,188743680~20971520,213909504~8388608,230686720~8388608,264241152~4194304,272629760~4194304,285212672~8388608,301989888~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.239 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~20971520,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,171966464~8388608,188743680~20971520,213909504~8388608,230686720~8388608,264241152~4194304,272629760~4194304,285212672~8388608,301989888~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.247 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~25165824,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~16777216,171966464~12582912,188743680~20971520,213909504~12582912,230686720~8388608,264241152~12582912,285212672~12582912,301989888~67108864,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.247 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 7 diff [0~12582912,29360128~20971520,54525952~8388608,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,163577856~4194304,171966464~8388608,184549376~37748736,230686720~8388608,264241152~4194304,272629760~8388608,285212672~8388608,301989888~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.248 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~20971520,54525952~8388608,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,163577856~4194304,171966464~8388608,184549376~37748736,230686720~8388608,264241152~4194304,272629760~8388608,285212672~8388608,301989888~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.258 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~33554432,79691776~29360128,113246208~8388608,125829120~4194304,138412032~16777216,163577856~62914560,230686720~8388608,264241152~16777216,285212672~12582912,301989888~67108864,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.258 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 8 diff [0~12582912,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~37748736,230686720~8388608,247463936~8388608,264241152~4194304,272629760~8388608,285212672~8388608,301989888~8388608,314572800~20971520,339738624~16777216,360710144~25165824,394264576~16777216] 2024-03-07T03:50:22.258 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~37748736,230686720~8388608,247463936~8388608,264241152~4194304,272629760~8388608,285212672~8388608,301989888~8388608,314572800~20971520,339738624~16777216,360710144~25165824,394264576~16777216] 2024-03-07T03:50:22.269 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~88080384,230686720~8388608,247463936~8388608,264241152~16777216,285212672~12582912,301989888~83886080,394264576~20971520] 2024-03-07T03:50:22.269 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 9 diff [0~16777216,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~37748736,230686720~8388608,247463936~8388608,260046848~8388608,272629760~20971520,301989888~8388608,314572800~20971520,339738624~46137344,394264576~16777216] 2024-03-07T03:50:22.269 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~16777216,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~37748736,230686720~8388608,247463936~8388608,260046848~8388608,272629760~20971520,301989888~8388608,314572800~20971520,339738624~46137344,394264576~16777216] 2024-03-07T03:50:22.280 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~20971520,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~88080384,230686720~8388608,247463936~8388608,260046848~37748736,301989888~83886080,394264576~20971520] 2024-03-07T03:50:22.280 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 10 diff [0~16777216,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~41943040,230686720~20971520,260046848~8388608,272629760~20971520,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.280 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~16777216,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~41943040,230686720~20971520,260046848~8388608,272629760~20971520,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.293 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~20971520,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~117440512,260046848~37748736,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.293 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 11 diff [0~25165824,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~79691776,260046848~33554432,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.293 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~33554432,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~79691776,260046848~33554432,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.306 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~62914560,71303168~37748736,113246208~8388608,125829120~4194304,138412032~117440512,260046848~37748736,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.306 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 12 diff [0~25165824,29360128~79691776,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~79691776,255852544~37748736,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.306 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~79691776,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~79691776,255852544~37748736,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.321 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~109051904,113246208~8388608,125829120~171966464,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.321 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 13 diff [0~25165824,29360128~79691776,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~121634816,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.321 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~79691776,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~121634816,301989888~33554432,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.340 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~109051904,113246208~8388608,125829120~171966464,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.340 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 14 diff [0~25165824,29360128~79691776,113246208~29360128,146800640~20971520,171966464~121634816,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.340 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~79691776,113246208~29360128,146800640~20971520,171966464~121634816,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.354 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~109051904,113246208~184549376,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.354 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 15 diff [0~25165824,29360128~79691776,113246208~29360128,146800640~20971520,171966464~125829120,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.354 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~79691776,113246208~29360128,146800640~20971520,171966464~125829120,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.371 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~109051904,113246208~184549376,301989888~88080384,394264576~20971520] 2024-03-07T03:50:22.371 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 16 diff [0~25165824,29360128~79691776,113246208~54525952,171966464~125829120,301989888~109051904] 2024-03-07T03:50:22.371 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~79691776,113246208~54525952,171966464~125829120,301989888~109051904] 2024-03-07T03:50:22.389 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~109051904,113246208~184549376,301989888~113246208] 2024-03-07T03:50:22.390 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 17 diff [0~167772160,171966464~125829120,301989888~109051904] 2024-03-07T03:50:22.390 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~125829120,301989888~109051904] 2024-03-07T03:50:22.410 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:22.410 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 18 diff [0~167772160,171966464~125829120,301989888~109051904] 2024-03-07T03:50:22.410 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~125829120,301989888~109051904] 2024-03-07T03:50:22.431 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:22.431 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to 19 diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:22.431 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:22.452 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:22.452 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 2 diff [29360128~4194304,41943040~8388608,104857600~4194304,146800640~8388608,171966464~8388608,272629760~4194304,314572800~4194304,343932928~4194304,360710144~4194304,402653184~4194304] 2024-03-07T03:50:22.452 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [29360128~4194304,41943040~8388608,104857600~4194304,146800640~8388608,171966464~8388608,272629760~4194304,314572800~4194304,343932928~4194304,360710144~4194304,402653184~4194304] 2024-03-07T03:50:22.455 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [29360128~4194304,41943040~12582912,104857600~4194304,146800640~8388608,171966464~12582912,272629760~4194304,314572800~4194304,343932928~8388608,360710144~8388608,402653184~4194304] 2024-03-07T03:50:22.455 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 3 diff [0~4194304,29360128~4194304,37748736~12582912,104857600~4194304,113246208~4194304,146800640~8388608,171966464~8388608,188743680~8388608,272629760~4194304,285212672~8388608,314572800~4194304,327155712~8388608,343932928~4194304,360710144~4194304,394264576~16777216] 2024-03-07T03:50:22.456 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,29360128~4194304,37748736~12582912,104857600~4194304,113246208~4194304,146800640~8388608,171966464~8388608,188743680~8388608,272629760~4194304,285212672~8388608,314572800~4194304,327155712~8388608,343932928~4194304,360710144~4194304,394264576~16777216] 2024-03-07T03:50:22.459 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,29360128~4194304,37748736~16777216,104857600~4194304,113246208~4194304,146800640~8388608,171966464~12582912,188743680~12582912,272629760~4194304,285212672~12582912,314572800~4194304,327155712~12582912,343932928~8388608,360710144~8388608,394264576~20971520] 2024-03-07T03:50:22.460 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 4 diff [0~4194304,8388608~8388608,29360128~4194304,37748736~12582912,100663296~8388608,113246208~4194304,146800640~8388608,171966464~8388608,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,314572800~4194304,327155712~8388608,343932928~4194304,360710144~4194304,373293056~8388608,394264576~16777216] 2024-03-07T03:50:22.460 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,8388608~8388608,29360128~4194304,37748736~12582912,100663296~8388608,113246208~4194304,146800640~8388608,171966464~8388608,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,314572800~4194304,327155712~8388608,343932928~4194304,360710144~4194304,373293056~8388608,394264576~16777216] 2024-03-07T03:50:22.465 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,8388608~8388608,29360128~4194304,37748736~16777216,100663296~8388608,113246208~4194304,146800640~8388608,171966464~12582912,188743680~20971520,230686720~8388608,272629760~4194304,285212672~12582912,314572800~4194304,327155712~12582912,343932928~8388608,360710144~8388608,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.465 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 5 diff [0~4194304,8388608~8388608,29360128~4194304,37748736~12582912,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,146800640~8388608,171966464~8388608,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,314572800~4194304,327155712~8388608,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.465 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,8388608~8388608,29360128~4194304,37748736~12582912,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,146800640~8388608,171966464~8388608,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,314572800~4194304,327155712~8388608,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.471 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,8388608~8388608,29360128~4194304,37748736~16777216,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,146800640~8388608,171966464~12582912,188743680~20971520,230686720~8388608,272629760~4194304,285212672~12582912,314572800~4194304,327155712~29360128,360710144~8388608,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.471 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 6 diff [0~12582912,29360128~4194304,37748736~12582912,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,171966464~8388608,188743680~16777216,213909504~8388608,230686720~4194304,264241152~4194304,272629760~4194304,285212672~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.471 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~4194304,37748736~12582912,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,171966464~8388608,188743680~16777216,213909504~8388608,230686720~4194304,264241152~4194304,272629760~4194304,285212672~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.479 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~4194304,37748736~16777216,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~16777216,171966464~12582912,188743680~20971520,213909504~12582912,230686720~8388608,264241152~12582912,285212672~12582912,314572800~41943040,360710144~8388608,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.479 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 7 diff [0~12582912,29360128~4194304,37748736~12582912,54525952~8388608,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,264241152~4194304,272629760~8388608,285212672~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.479 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~4194304,37748736~12582912,54525952~8388608,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,146800640~8388608,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,264241152~4194304,272629760~8388608,285212672~8388608,314572800~20971520,339738624~8388608,352321536~4194304,360710144~8388608,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.487 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~4194304,37748736~25165824,79691776~29360128,113246208~8388608,125829120~4194304,138412032~16777216,163577856~62914560,230686720~8388608,264241152~16777216,285212672~12582912,314572800~41943040,360710144~8388608,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.488 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 8 diff [0~12582912,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,264241152~4194304,272629760~8388608,285212672~8388608,314572800~20971520,339738624~16777216,360710144~25165824,394264576~16777216] 2024-03-07T03:50:22.488 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,264241152~4194304,272629760~8388608,285212672~8388608,314572800~20971520,339738624~16777216,360710144~25165824,394264576~16777216] 2024-03-07T03:50:22.497 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~88080384,230686720~8388608,247463936~8388608,264241152~16777216,285212672~12582912,314572800~41943040,360710144~25165824,394264576~20971520] 2024-03-07T03:50:22.497 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 9 diff [0~16777216,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,260046848~8388608,272629760~20971520,314572800~20971520,339738624~46137344,394264576~16777216] 2024-03-07T03:50:22.497 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~16777216,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,260046848~8388608,272629760~20971520,314572800~20971520,339738624~46137344,394264576~16777216] 2024-03-07T03:50:22.508 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~20971520,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~88080384,230686720~8388608,247463936~8388608,260046848~37748736,314572800~71303168,394264576~20971520] 2024-03-07T03:50:22.508 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 10 diff [0~16777216,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~20971520,209715200~16777216,230686720~20971520,260046848~8388608,272629760~20971520,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.508 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~16777216,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~8388608,184549376~20971520,209715200~16777216,230686720~20971520,260046848~8388608,272629760~20971520,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.520 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~20971520,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~117440512,260046848~37748736,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.520 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 11 diff [0~25165824,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~33554432,209715200~41943040,260046848~33554432,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.520 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,146800640~12582912,163577856~4194304,171966464~33554432,209715200~41943040,260046848~33554432,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.532 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~25165824,71303168~37748736,113246208~8388608,125829120~4194304,138412032~117440512,260046848~37748736,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.532 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 12 diff [0~25165824,29360128~4194304,37748736~71303168,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~33554432,209715200~41943040,255852544~37748736,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.532 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~71303168,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~33554432,209715200~41943040,255852544~37748736,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.545 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~71303168,113246208~8388608,125829120~171966464,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.546 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 13 diff [0~25165824,29360128~4194304,37748736~71303168,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~33554432,209715200~83886080,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.546 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~71303168,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~33554432,209715200~83886080,306184192~29360128,339738624~50331648,394264576~16777216] 2024-03-07T03:50:22.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:22 smithi148 bash[79819]: cluster 2024-03-07T03:50:21.147463+0000 mon.a (mon.0) 1932 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2024-03-07T03:50:22.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:22 smithi148 bash[79819]: cluster 2024-03-07T03:50:21.260824+0000 mgr.y (mgr.34101) 1092 : cluster 0 pgmap v1787: 105 pgs: 105 active+clean; 754 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 112 KiB/s rd, 44 MiB/s wr, 201 op/s 2024-03-07T03:50:22.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:22 smithi148 bash[82222]: cluster 2024-03-07T03:50:21.147463+0000 mon.a (mon.0) 1932 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2024-03-07T03:50:22.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:22 smithi148 bash[82222]: cluster 2024-03-07T03:50:21.260824+0000 mgr.y (mgr.34101) 1092 : cluster 0 pgmap v1787: 105 pgs: 105 active+clean; 754 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 112 KiB/s rd, 44 MiB/s wr, 201 op/s 2024-03-07T03:50:22.560 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~71303168,113246208~8388608,125829120~171966464,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.560 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 14 diff [0~25165824,29360128~4194304,37748736~71303168,113246208~29360128,146800640~20971520,171966464~33554432,209715200~83886080,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.560 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~71303168,113246208~29360128,146800640~20971520,171966464~33554432,209715200~83886080,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:22 smithi195 bash[66542]: cluster 2024-03-07T03:50:21.147463+0000 mon.a (mon.0) 1932 : cluster 0 osdmap e1076: 8 total, 8 up, 8 in 2024-03-07T03:50:22.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:22 smithi195 bash[66542]: cluster 2024-03-07T03:50:21.260824+0000 mgr.y (mgr.34101) 1092 : cluster 0 pgmap v1787: 105 pgs: 105 active+clean; 754 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 112 KiB/s rd, 44 MiB/s wr, 201 op/s 2024-03-07T03:50:22.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:50:22 smithi195 bash[59134]: ts=2024-03-07T03:50:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:22.575 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~71303168,113246208~184549376,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.575 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 15 diff [0~25165824,29360128~4194304,37748736~71303168,113246208~29360128,146800640~20971520,171966464~33554432,209715200~88080384,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.575 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~71303168,113246208~29360128,146800640~20971520,171966464~33554432,209715200~88080384,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.592 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~71303168,113246208~184549376,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.592 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 16 diff [0~25165824,29360128~4194304,37748736~71303168,113246208~54525952,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.592 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~71303168,113246208~54525952,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.609 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~71303168,113246208~184549376,306184192~109051904] 2024-03-07T03:50:22.609 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 17 diff [0~33554432,37748736~130023424,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.609 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.627 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:22.627 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 18 diff [0~33554432,37748736~130023424,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.628 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.646 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:22.646 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to 19 diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.647 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.667 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:22.667 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 3 diff [0~4194304,37748736~8388608,113246208~4194304,171966464~4194304,188743680~8388608,285212672~8388608,327155712~8388608,394264576~16777216] 2024-03-07T03:50:22.667 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,37748736~8388608,113246208~4194304,171966464~4194304,188743680~8388608,285212672~8388608,327155712~8388608,394264576~16777216] 2024-03-07T03:50:22.670 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,37748736~8388608,113246208~4194304,171966464~4194304,188743680~12582912,285212672~12582912,327155712~12582912,394264576~20971520] 2024-03-07T03:50:22.670 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 4 diff [0~4194304,8388608~8388608,37748736~8388608,100663296~8388608,113246208~4194304,171966464~4194304,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,327155712~8388608,373293056~8388608,394264576~16777216] 2024-03-07T03:50:22.670 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,8388608~8388608,37748736~8388608,100663296~8388608,113246208~4194304,171966464~4194304,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,327155712~8388608,373293056~8388608,394264576~16777216] 2024-03-07T03:50:22.674 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,8388608~8388608,37748736~8388608,100663296~8388608,113246208~4194304,171966464~4194304,188743680~20971520,230686720~8388608,272629760~4194304,285212672~12582912,327155712~12582912,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.674 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 5 diff [0~4194304,8388608~8388608,37748736~8388608,50331648~4194304,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,171966464~4194304,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,327155712~8388608,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.674 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~4194304,8388608~8388608,37748736~8388608,50331648~4194304,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,171966464~4194304,188743680~16777216,230686720~4194304,272629760~4194304,285212672~8388608,327155712~8388608,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.680 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~4194304,8388608~8388608,37748736~8388608,50331648~4194304,58720256~4194304,79691776~8388608,100663296~8388608,113246208~8388608,171966464~4194304,188743680~20971520,230686720~8388608,272629760~4194304,285212672~12582912,327155712~16777216,352321536~4194304,364904448~4194304,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.680 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 6 diff [0~12582912,29360128~4194304,37748736~8388608,50331648~4194304,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~4194304,171966464~4194304,188743680~16777216,213909504~8388608,230686720~4194304,264241152~4194304,272629760~4194304,285212672~8388608,318767104~16777216,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.680 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~4194304,37748736~8388608,50331648~4194304,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~4194304,171966464~4194304,188743680~16777216,213909504~8388608,230686720~4194304,264241152~4194304,272629760~4194304,285212672~8388608,318767104~16777216,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.686 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~4194304,37748736~8388608,50331648~4194304,58720256~4194304,79691776~12582912,100663296~8388608,113246208~8388608,125829120~4194304,138412032~8388608,171966464~4194304,188743680~20971520,213909504~12582912,230686720~8388608,264241152~12582912,285212672~12582912,318767104~25165824,352321536~4194304,364904448~4194304,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.686 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 7 diff [0~12582912,29360128~4194304,37748736~8388608,50331648~12582912,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,163577856~4194304,171966464~4194304,184549376~20971520,209715200~12582912,230686720~4194304,264241152~4194304,272629760~8388608,285212672~8388608,318767104~16777216,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.686 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~4194304,37748736~8388608,50331648~12582912,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,163577856~4194304,171966464~4194304,184549376~20971520,209715200~12582912,230686720~4194304,264241152~4194304,272629760~8388608,285212672~8388608,318767104~16777216,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~16777216] 2024-03-07T03:50:22.693 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~4194304,37748736~8388608,50331648~12582912,79691776~29360128,113246208~8388608,125829120~4194304,138412032~8388608,163577856~12582912,184549376~41943040,230686720~8388608,264241152~16777216,285212672~12582912,318767104~25165824,352321536~4194304,364904448~4194304,373293056~12582912,394264576~20971520] 2024-03-07T03:50:22.693 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 8 diff [0~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~4194304,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,264241152~4194304,272629760~8388608,285212672~8388608,318767104~16777216,339738624~4194304,348127232~8388608,364904448~20971520,394264576~16777216] 2024-03-07T03:50:22.693 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~4194304,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,264241152~4194304,272629760~8388608,285212672~8388608,318767104~16777216,339738624~4194304,348127232~8388608,364904448~20971520,394264576~16777216] 2024-03-07T03:50:22.701 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~20971520,184549376~41943040,230686720~8388608,247463936~8388608,264241152~16777216,285212672~12582912,318767104~25165824,348127232~8388608,364904448~20971520,394264576~20971520] 2024-03-07T03:50:22.701 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 9 diff [0~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,260046848~8388608,272629760~20971520,318767104~16777216,339738624~4194304,348127232~37748736,394264576~16777216] 2024-03-07T03:50:22.701 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~20971520,209715200~12582912,230686720~4194304,247463936~8388608,260046848~8388608,272629760~20971520,318767104~16777216,339738624~4194304,348127232~37748736,394264576~16777216] 2024-03-07T03:50:22.710 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~25165824,184549376~41943040,230686720~8388608,247463936~8388608,260046848~37748736,318767104~25165824,348127232~37748736,394264576~20971520] 2024-03-07T03:50:22.710 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 10 diff [0~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~20971520,209715200~16777216,230686720~20971520,260046848~8388608,272629760~20971520,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.710 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~20971520,209715200~16777216,230686720~20971520,260046848~8388608,272629760~20971520,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.721 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~25165824,184549376~71303168,260046848~37748736,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.721 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 11 diff [0~25165824,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~33554432,209715200~41943040,260046848~33554432,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.721 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~33554432,209715200~41943040,260046848~33554432,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.732 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~100663296,260046848~37748736,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.732 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 12 diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~33554432,209715200~41943040,255852544~37748736,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.733 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~33554432,209715200~41943040,255852544~37748736,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.747 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~8388608,50331648~58720256,113246208~8388608,125829120~171966464,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.747 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 13 diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~33554432,209715200~83886080,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.747 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~33554432,209715200~83886080,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.757 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~8388608,50331648~58720256,113246208~8388608,125829120~171966464,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.757 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 14 diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~33554432,209715200~83886080,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.757 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~33554432,209715200~83886080,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.770 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~8388608,50331648~58720256,113246208~184549376,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.770 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 15 diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~33554432,209715200~88080384,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.770 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~33554432,209715200~88080384,306184192~37748736,348127232~41943040,394264576~20971520] 2024-03-07T03:50:22.787 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~8388608,50331648~58720256,113246208~184549376,306184192~83886080,394264576~20971520] 2024-03-07T03:50:22.787 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 16 diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~54525952,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.787 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~25165824,29360128~4194304,37748736~8388608,50331648~58720256,113246208~54525952,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.803 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~8388608,50331648~58720256,113246208~184549376,306184192~109051904] 2024-03-07T03:50:22.803 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 17 diff [0~33554432,37748736~8388608,50331648~117440512,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.803 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~8388608,50331648~117440512,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.821 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~8388608,50331648~247463936,306184192~109051904] 2024-03-07T03:50:22.821 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 18 diff [0~33554432,37748736~130023424,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.821 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~88080384,306184192~104857600] 2024-03-07T03:50:22.839 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:22.839 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to 19 diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.839 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.859 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:22.859 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 4 diff [8388608~8388608,37748736~8388608,100663296~8388608,197132288~8388608,230686720~4194304,272629760~4194304,289406976~4194304,373293056~8388608] 2024-03-07T03:50:22.859 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [8388608~8388608,37748736~8388608,100663296~8388608,197132288~8388608,230686720~4194304,272629760~4194304,289406976~4194304,373293056~8388608] 2024-03-07T03:50:22.862 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [8388608~8388608,37748736~8388608,100663296~8388608,197132288~12582912,230686720~8388608,272629760~4194304,289406976~8388608,373293056~12582912] 2024-03-07T03:50:22.862 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 5 diff [8388608~8388608,37748736~8388608,50331648~4194304,58720256~4194304,79691776~8388608,100663296~8388608,117440512~4194304,197132288~8388608,230686720~4194304,272629760~4194304,289406976~4194304,327155712~4194304,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.862 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [8388608~8388608,37748736~8388608,50331648~4194304,58720256~4194304,79691776~8388608,100663296~8388608,117440512~4194304,197132288~8388608,230686720~4194304,272629760~4194304,289406976~4194304,327155712~4194304,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.866 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [8388608~8388608,37748736~8388608,50331648~4194304,58720256~4194304,79691776~8388608,100663296~8388608,117440512~4194304,197132288~12582912,230686720~8388608,272629760~4194304,289406976~8388608,327155712~4194304,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.866 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 6 diff [4194304~8388608,29360128~4194304,37748736~8388608,50331648~4194304,58720256~4194304,79691776~12582912,100663296~8388608,117440512~4194304,125829120~4194304,138412032~4194304,197132288~8388608,213909504~8388608,230686720~4194304,264241152~4194304,272629760~4194304,289406976~4194304,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.866 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~8388608,50331648~4194304,58720256~4194304,79691776~12582912,100663296~8388608,117440512~4194304,125829120~4194304,138412032~4194304,197132288~8388608,213909504~8388608,230686720~4194304,264241152~4194304,272629760~4194304,289406976~4194304,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.872 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,50331648~4194304,58720256~4194304,79691776~12582912,100663296~8388608,117440512~4194304,125829120~4194304,138412032~8388608,197132288~12582912,213909504~12582912,230686720~8388608,264241152~12582912,289406976~8388608,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.872 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 7 diff [4194304~8388608,29360128~4194304,37748736~8388608,50331648~12582912,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,230686720~4194304,264241152~4194304,272629760~8388608,289406976~4194304,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.872 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~8388608,50331648~12582912,79691776~29360128,113246208~8388608,125829120~4194304,138412032~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,230686720~4194304,264241152~4194304,272629760~8388608,289406976~4194304,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.878 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,79691776~29360128,113246208~8388608,125829120~4194304,138412032~8388608,163577856~8388608,184549376~8388608,197132288~29360128,230686720~8388608,264241152~16777216,289406976~8388608,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:22.878 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 8 diff [4194304~8388608,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,230686720~4194304,247463936~8388608,264241152~4194304,272629760~8388608,289406976~4194304,318767104~12582912,339738624~4194304,348127232~8388608,364904448~20971520,394264576~4194304,402653184~8388608] 2024-03-07T03:50:22.878 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,230686720~4194304,247463936~8388608,264241152~4194304,272629760~8388608,289406976~4194304,318767104~12582912,339738624~4194304,348127232~8388608,364904448~20971520,394264576~4194304,402653184~8388608] 2024-03-07T03:50:22.885 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~16777216,184549376~8388608,197132288~29360128,230686720~8388608,247463936~8388608,264241152~16777216,289406976~8388608,318767104~12582912,339738624~4194304,348127232~8388608,364904448~20971520,394264576~4194304,402653184~8388608] 2024-03-07T03:50:22.885 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 9 diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,230686720~4194304,247463936~8388608,260046848~8388608,272629760~12582912,289406976~4194304,318767104~12582912,339738624~4194304,348127232~37748736,394264576~4194304,402653184~8388608] 2024-03-07T03:50:22.885 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,230686720~4194304,247463936~8388608,260046848~8388608,272629760~12582912,289406976~4194304,318767104~12582912,339738624~4194304,348127232~37748736,394264576~4194304,402653184~8388608] 2024-03-07T03:50:22.893 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~25165824,184549376~8388608,197132288~29360128,230686720~8388608,247463936~8388608,260046848~25165824,289406976~8388608,318767104~12582912,339738624~4194304,348127232~37748736,394264576~4194304,402653184~8388608] 2024-03-07T03:50:22.894 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 10 diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,230686720~20971520,260046848~8388608,272629760~12582912,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.894 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,230686720~20971520,260046848~8388608,272629760~12582912,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.904 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~25165824,184549376~8388608,197132288~58720256,260046848~25165824,289406976~8388608,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.904 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 11 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~41943040,260046848~25165824,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.904 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~41943040,260046848~25165824,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.915 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~37748736,197132288~58720256,260046848~25165824,289406976~8388608,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.915 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 12 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~41943040,255852544~29360128,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.915 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~41943040,255852544~29360128,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.927 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~8388608,125829120~67108864,197132288~88080384,289406976~8388608,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.927 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 13 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~16777216,197132288~8388608,209715200~75497472,289406976~4194304,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.927 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~16777216,197132288~8388608,209715200~75497472,289406976~4194304,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.938 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~8388608,125829120~67108864,197132288~88080384,289406976~8388608,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.938 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 14 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~75497472,289406976~4194304,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.938 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~75497472,289406976~4194304,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.951 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~88080384,289406976~8388608,306184192~83886080,394264576~16777216] 2024-03-07T03:50:22.952 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 15 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~83886080,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.952 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~83886080,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:22.967 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~100663296,306184192~83886080,394264576~16777216] 2024-03-07T03:50:22.967 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 16 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~54525952,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.967 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~58720256,113246208~54525952,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.981 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~100663296,306184192~104857600] 2024-03-07T03:50:22.981 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 17 diff [4194304~29360128,37748736~8388608,50331648~117440512,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.981 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~8388608,50331648~117440512,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.997 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~142606336,197132288~100663296,306184192~104857600] 2024-03-07T03:50:22.997 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 18 diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:22.997 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:23.013 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.014 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to 19 diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:23.014 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:23.032 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.033 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 5 diff [50331648~4194304,58720256~4194304,79691776~8388608,117440512~4194304,327155712~4194304,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.033 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [50331648~4194304,58720256~4194304,79691776~8388608,117440512~4194304,327155712~4194304,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.037 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [50331648~4194304,58720256~4194304,79691776~8388608,117440512~4194304,327155712~4194304,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.037 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 6 diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~4194304,58720256~4194304,79691776~12582912,117440512~4194304,125829120~4194304,138412032~4194304,213909504~8388608,264241152~4194304,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.037 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~4194304,58720256~4194304,79691776~12582912,117440512~4194304,125829120~4194304,138412032~4194304,213909504~8388608,264241152~4194304,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.040 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,50331648~4194304,58720256~4194304,79691776~12582912,117440512~4194304,125829120~4194304,138412032~8388608,213909504~12582912,264241152~8388608,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.040 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 7 diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~12582912,79691776~20971520,113246208~8388608,125829120~4194304,138412032~4194304,163577856~4194304,184549376~4194304,209715200~12582912,264241152~4194304,272629760~8388608,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.040 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~12582912,79691776~20971520,113246208~8388608,125829120~4194304,138412032~4194304,163577856~4194304,184549376~4194304,209715200~12582912,264241152~4194304,272629760~8388608,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~8388608,394264576~4194304] 2024-03-07T03:50:23.046 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,79691776~20971520,113246208~8388608,125829120~4194304,138412032~8388608,163577856~8388608,184549376~8388608,209715200~16777216,264241152~16777216,318767104~12582912,339738624~4194304,352321536~4194304,364904448~4194304,373293056~12582912,394264576~4194304] 2024-03-07T03:50:23.046 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 8 diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,264241152~4194304,272629760~8388608,318767104~12582912,339738624~4194304,348127232~8388608,364904448~16777216,394264576~4194304,402653184~8388608] 2024-03-07T03:50:23.046 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,264241152~4194304,272629760~8388608,318767104~12582912,339738624~4194304,348127232~8388608,364904448~16777216,394264576~4194304,402653184~8388608] 2024-03-07T03:50:23.052 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~8388608,155189248~16777216,184549376~8388608,197132288~29360128,247463936~8388608,264241152~16777216,318767104~12582912,339738624~4194304,348127232~8388608,364904448~20971520,394264576~4194304,402653184~8388608] 2024-03-07T03:50:23.052 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 9 diff [4194304~12582912,29360128~4194304,37748736~4194304,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,260046848~8388608,272629760~12582912,318767104~12582912,339738624~4194304,348127232~33554432,394264576~4194304,402653184~8388608] 2024-03-07T03:50:23.052 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~4194304,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,260046848~8388608,272629760~12582912,318767104~12582912,339738624~4194304,348127232~33554432,394264576~4194304,402653184~8388608] 2024-03-07T03:50:23.059 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~8388608,155189248~25165824,184549376~8388608,197132288~29360128,247463936~8388608,260046848~25165824,318767104~12582912,339738624~4194304,348127232~37748736,394264576~4194304,402653184~8388608] 2024-03-07T03:50:23.059 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 10 diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,234881024~20971520,260046848~8388608,272629760~12582912,306184192~25165824,339738624~4194304,348127232~33554432,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.060 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,234881024~20971520,260046848~8388608,272629760~12582912,306184192~25165824,339738624~4194304,348127232~33554432,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.069 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~8388608,125829120~4194304,138412032~8388608,155189248~25165824,184549376~8388608,197132288~33554432,234881024~20971520,260046848~25165824,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.069 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 11 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~33554432,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~20971520,260046848~25165824,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.069 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~33554432,113246208~8388608,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~20971520,260046848~25165824,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.078 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~12582912,71303168~37748736,113246208~8388608,125829120~4194304,138412032~8388608,155189248~37748736,197132288~58720256,260046848~25165824,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.078 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 12 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~8388608,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.079 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~8388608,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.089 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~8388608,125829120~67108864,197132288~88080384,289406976~4194304,306184192~25165824,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.089 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 13 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.089 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~8388608,125829120~8388608,138412032~4194304,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.100 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~8388608,125829120~67108864,197132288~88080384,289406976~4194304,306184192~29360128,339738624~4194304,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.100 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 14 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.100 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.113 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~88080384,289406976~4194304,306184192~83886080,394264576~16777216] 2024-03-07T03:50:23.113 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 15 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~58720256,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.113 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~58720256,306184192~37748736,348127232~41943040,394264576~16777216] 2024-03-07T03:50:23.126 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~96468992,306184192~83886080,394264576~16777216] 2024-03-07T03:50:23.126 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 16 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~54525952,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.127 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~54525952,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.141 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.141 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 17 diff [4194304~29360128,37748736~8388608,50331648~54525952,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.141 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~8388608,50331648~54525952,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.156 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~142606336,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.156 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 18 diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.156 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.174 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.174 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to 19 diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.174 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.190 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.190 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 6 diff [4194304~8388608,29360128~4194304,37748736~4194304,83886080~4194304,125829120~4194304,138412032~4194304,213909504~8388608,264241152~4194304,318767104~8388608] 2024-03-07T03:50:23.191 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~4194304,83886080~4194304,125829120~4194304,138412032~4194304,213909504~8388608,264241152~4194304,318767104~8388608] 2024-03-07T03:50:23.194 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,83886080~8388608,125829120~4194304,138412032~8388608,213909504~12582912,264241152~8388608,318767104~8388608] 2024-03-07T03:50:23.194 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 7 diff [4194304~8388608,29360128~4194304,37748736~4194304,54525952~8388608,83886080~4194304,92274688~8388608,113246208~4194304,125829120~4194304,138412032~4194304,163577856~4194304,184549376~4194304,209715200~12582912,264241152~4194304,272629760~8388608,318767104~8388608,339738624~4194304,377487360~4194304] 2024-03-07T03:50:23.194 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~4194304,54525952~8388608,83886080~4194304,92274688~8388608,113246208~4194304,125829120~4194304,138412032~4194304,163577856~4194304,184549376~4194304,209715200~12582912,264241152~4194304,272629760~8388608,318767104~8388608,339738624~4194304,377487360~4194304] 2024-03-07T03:50:23.198 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,54525952~8388608,83886080~16777216,113246208~4194304,125829120~4194304,138412032~8388608,163577856~8388608,184549376~8388608,209715200~16777216,264241152~16777216,318767104~8388608,339738624~4194304,377487360~8388608] 2024-03-07T03:50:23.198 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 8 diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~12582912,71303168~16777216,92274688~8388608,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,264241152~4194304,272629760~8388608,318767104~8388608,339738624~4194304,348127232~8388608,364904448~4194304,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.198 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~8388608,29360128~4194304,37748736~4194304,50331648~12582912,71303168~16777216,92274688~8388608,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,264241152~4194304,272629760~8388608,318767104~8388608,339738624~4194304,348127232~8388608,364904448~4194304,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.204 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~4194304,125829120~4194304,138412032~8388608,155189248~16777216,184549376~8388608,197132288~29360128,247463936~8388608,264241152~16777216,318767104~8388608,339738624~4194304,348127232~8388608,364904448~8388608,377487360~8388608,402653184~8388608] 2024-03-07T03:50:23.204 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 9 diff [4194304~12582912,29360128~4194304,37748736~4194304,50331648~12582912,71303168~29360128,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,260046848~8388608,272629760~12582912,318767104~8388608,339738624~4194304,348127232~20971520,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.204 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~4194304,50331648~12582912,71303168~29360128,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,260046848~8388608,272629760~12582912,318767104~8388608,339738624~4194304,348127232~20971520,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.210 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~4194304,125829120~4194304,138412032~8388608,155189248~25165824,184549376~8388608,197132288~29360128,247463936~8388608,260046848~25165824,318767104~8388608,339738624~4194304,348127232~25165824,377487360~8388608,402653184~8388608] 2024-03-07T03:50:23.211 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 10 diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,234881024~20971520,260046848~8388608,272629760~12582912,306184192~20971520,339738624~4194304,348127232~20971520,377487360~4194304,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.211 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,234881024~20971520,260046848~8388608,272629760~12582912,306184192~20971520,339738624~4194304,348127232~20971520,377487360~4194304,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.218 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,29360128~4194304,37748736~8388608,50331648~12582912,71303168~29360128,113246208~4194304,125829120~4194304,138412032~8388608,155189248~25165824,184549376~8388608,197132288~33554432,234881024~20971520,260046848~25165824,306184192~20971520,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.218 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 11 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~33554432,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~20971520,260046848~25165824,306184192~20971520,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.218 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~12582912,71303168~33554432,113246208~4194304,125829120~4194304,138412032~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~20971520,260046848~25165824,306184192~20971520,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.227 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~12582912,71303168~37748736,113246208~4194304,125829120~4194304,138412032~8388608,155189248~37748736,197132288~58720256,260046848~25165824,306184192~20971520,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.227 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 12 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~4194304,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~20971520,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.227 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~4194304,125829120~8388608,138412032~4194304,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~20971520,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.235 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~4194304,125829120~67108864,197132288~88080384,289406976~4194304,306184192~20971520,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.235 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 13 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~4194304,125829120~8388608,138412032~4194304,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~29360128,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.236 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~4194304,125829120~8388608,138412032~4194304,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~29360128,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.246 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~4194304,125829120~67108864,197132288~88080384,289406976~4194304,306184192~29360128,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.246 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 14 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~4194304,121634816~20971520,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~37748736,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.246 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~4194304,121634816~20971520,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~37748736,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.258 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~4194304,121634816~71303168,197132288~88080384,289406976~4194304,306184192~67108864,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.259 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 15 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~58720256,306184192~37748736,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.259 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~29360128,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~58720256,306184192~37748736,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.271 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~96468992,306184192~83886080,394264576~16777216] 2024-03-07T03:50:23.271 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 16 diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~54525952,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~67108864,377487360~33554432] 2024-03-07T03:50:23.271 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,29360128~4194304,37748736~8388608,50331648~54525952,113246208~54525952,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~67108864,377487360~33554432] 2024-03-07T03:50:23.285 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~58720256,113246208~79691776,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.285 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 17 diff [4194304~29360128,37748736~8388608,50331648~54525952,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.285 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~8388608,50331648~54525952,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.299 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~8388608,50331648~142606336,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.300 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 18 diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.300 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.316 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.316 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to 19 diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.316 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:23.333 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.333 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 7 diff [54525952~8388608,92274688~8388608,113246208~4194304,163577856~4194304,184549376~4194304,209715200~12582912,272629760~8388608,339738624~4194304,377487360~4194304] 2024-03-07T03:50:23.333 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [54525952~8388608,92274688~8388608,113246208~4194304,163577856~4194304,184549376~4194304,209715200~12582912,272629760~8388608,339738624~4194304,377487360~4194304] 2024-03-07T03:50:23.336 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [54525952~8388608,92274688~8388608,113246208~4194304,163577856~8388608,184549376~8388608,209715200~16777216,272629760~8388608,339738624~4194304,377487360~8388608] 2024-03-07T03:50:23.336 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 8 diff [50331648~12582912,71303168~12582912,92274688~8388608,113246208~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,272629760~8388608,339738624~4194304,348127232~8388608,364904448~4194304,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.336 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [50331648~12582912,71303168~12582912,92274688~8388608,113246208~4194304,155189248~4194304,163577856~4194304,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,272629760~8388608,339738624~4194304,348127232~8388608,364904448~4194304,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.340 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [50331648~12582912,71303168~16777216,92274688~8388608,113246208~4194304,155189248~16777216,184549376~8388608,197132288~29360128,247463936~8388608,272629760~8388608,339738624~4194304,348127232~8388608,364904448~8388608,377487360~8388608,402653184~8388608] 2024-03-07T03:50:23.340 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 9 diff [4194304~12582912,50331648~12582912,71303168~12582912,88080384~12582912,113246208~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,260046848~8388608,272629760~12582912,339738624~4194304,348127232~20971520,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.340 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,50331648~12582912,71303168~12582912,88080384~12582912,113246208~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~12582912,247463936~8388608,260046848~8388608,272629760~12582912,339738624~4194304,348127232~20971520,377487360~4194304,402653184~8388608] 2024-03-07T03:50:23.346 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,50331648~12582912,71303168~29360128,113246208~4194304,155189248~25165824,184549376~8388608,197132288~29360128,247463936~8388608,260046848~25165824,339738624~4194304,348127232~25165824,377487360~8388608,402653184~8388608] 2024-03-07T03:50:23.346 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 10 diff [4194304~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~12582912,113246208~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,234881024~20971520,260046848~8388608,272629760~12582912,306184192~12582912,339738624~4194304,348127232~20971520,377487360~4194304,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.346 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,41943040~4194304,50331648~12582912,71303168~12582912,88080384~12582912,113246208~4194304,155189248~4194304,163577856~4194304,171966464~8388608,184549376~4194304,197132288~8388608,209715200~16777216,234881024~20971520,260046848~8388608,272629760~12582912,306184192~12582912,339738624~4194304,348127232~20971520,377487360~4194304,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.353 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,41943040~4194304,50331648~12582912,71303168~29360128,113246208~4194304,155189248~25165824,184549376~8388608,197132288~33554432,234881024~20971520,260046848~25165824,306184192~16777216,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.353 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 11 diff [4194304~20971520,41943040~4194304,50331648~12582912,71303168~33554432,113246208~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~20971520,260046848~25165824,306184192~16777216,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.353 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~12582912,71303168~33554432,113246208~4194304,155189248~4194304,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~20971520,260046848~25165824,306184192~16777216,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.360 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~12582912,71303168~37748736,113246208~4194304,155189248~37748736,197132288~58720256,260046848~25165824,306184192~20971520,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.360 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 12 diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~4194304,125829120~8388608,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~16777216,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.361 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~4194304,125829120~8388608,146800640~12582912,163577856~4194304,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~16777216,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.369 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,113246208~4194304,125829120~12582912,146800640~46137344,197132288~88080384,289406976~4194304,306184192~20971520,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.369 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 13 diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~4194304,125829120~8388608,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~16777216,327155712~8388608,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.369 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~4194304,125829120~8388608,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~16777216,327155712~8388608,339738624~4194304,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.378 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,113246208~4194304,125829120~12582912,146800640~46137344,197132288~88080384,289406976~4194304,306184192~29360128,339738624~4194304,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.378 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 14 diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~4194304,121634816~16777216,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.378 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~4194304,121634816~16777216,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~50331648,289406976~4194304,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.389 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,113246208~4194304,121634816~16777216,146800640~46137344,197132288~88080384,289406976~4194304,306184192~67108864,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.389 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 15 diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~25165824,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~58720256,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.389 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~25165824,146800640~20971520,171966464~16777216,197132288~8388608,209715200~20971520,234881024~58720256,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.400 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,113246208~25165824,146800640~46137344,197132288~96468992,306184192~83886080,394264576~16777216] 2024-03-07T03:50:23.400 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 16 diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~54525952,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~46137344,377487360~33554432] 2024-03-07T03:50:23.400 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,113246208~54525952,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~46137344,377487360~33554432] 2024-03-07T03:50:23.413 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,113246208~79691776,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.413 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 17 diff [4194304~25165824,41943040~4194304,50331648~54525952,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.413 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~4194304,50331648~54525952,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.425 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~142606336,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.426 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 18 diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.426 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.440 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.440 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to 19 diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.440 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.456 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.456 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 8 diff [50331648~8388608,71303168~12582912,155189248~4194304,197132288~8388608,247463936~8388608,348127232~8388608,364904448~4194304,402653184~8388608] 2024-03-07T03:50:23.456 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [50331648~8388608,71303168~12582912,155189248~4194304,197132288~8388608,247463936~8388608,348127232~8388608,364904448~4194304,402653184~8388608] 2024-03-07T03:50:23.459 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [50331648~8388608,71303168~16777216,155189248~8388608,197132288~12582912,247463936~8388608,348127232~8388608,364904448~8388608,402653184~8388608] 2024-03-07T03:50:23.459 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 9 diff [4194304~12582912,50331648~8388608,71303168~12582912,88080384~8388608,155189248~4194304,171966464~8388608,197132288~8388608,247463936~8388608,260046848~8388608,276824064~8388608,348127232~20971520,402653184~8388608] 2024-03-07T03:50:23.459 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,50331648~8388608,71303168~12582912,88080384~8388608,155189248~4194304,171966464~8388608,197132288~8388608,247463936~8388608,260046848~8388608,276824064~8388608,348127232~20971520,402653184~8388608] 2024-03-07T03:50:23.464 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,50331648~8388608,71303168~29360128,155189248~8388608,171966464~8388608,197132288~12582912,247463936~8388608,260046848~12582912,276824064~8388608,348127232~25165824,402653184~8388608] 2024-03-07T03:50:23.464 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 10 diff [4194304~12582912,41943040~4194304,50331648~8388608,71303168~12582912,88080384~8388608,155189248~4194304,171966464~8388608,197132288~8388608,213909504~4194304,222298112~4194304,234881024~20971520,260046848~8388608,276824064~8388608,306184192~12582912,348127232~20971520,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.464 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,41943040~4194304,50331648~8388608,71303168~12582912,88080384~8388608,155189248~4194304,171966464~8388608,197132288~8388608,213909504~4194304,222298112~4194304,234881024~20971520,260046848~8388608,276824064~8388608,306184192~12582912,348127232~20971520,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.470 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,41943040~4194304,50331648~8388608,71303168~29360128,155189248~8388608,171966464~8388608,197132288~12582912,213909504~16777216,234881024~20971520,260046848~12582912,276824064~8388608,306184192~16777216,348127232~25165824,385875968~4194304,394264576~16777216] 2024-03-07T03:50:23.470 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 11 diff [4194304~20971520,41943040~4194304,50331648~8388608,71303168~25165824,100663296~4194304,155189248~4194304,171966464~12582912,197132288~8388608,213909504~4194304,222298112~8388608,234881024~20971520,260046848~25165824,306184192~16777216,348127232~20971520,381681664~8388608,394264576~16777216] 2024-03-07T03:50:23.470 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~8388608,71303168~25165824,100663296~4194304,155189248~4194304,171966464~12582912,197132288~8388608,213909504~4194304,222298112~8388608,234881024~20971520,260046848~25165824,306184192~16777216,348127232~20971520,381681664~8388608,394264576~16777216] 2024-03-07T03:50:23.476 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~8388608,71303168~37748736,155189248~8388608,171966464~16777216,197132288~12582912,213909504~41943040,260046848~25165824,306184192~20971520,348127232~25165824,381681664~8388608,394264576~16777216] 2024-03-07T03:50:23.477 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 12 diff [4194304~20971520,41943040~4194304,50331648~46137344,100663296~4194304,125829120~8388608,146800640~12582912,171966464~12582912,197132288~8388608,213909504~4194304,222298112~8388608,234881024~50331648,289406976~4194304,306184192~16777216,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.477 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~46137344,100663296~4194304,125829120~8388608,146800640~12582912,171966464~12582912,197132288~8388608,213909504~4194304,222298112~8388608,234881024~50331648,289406976~4194304,306184192~16777216,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.484 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,125829120~12582912,146800640~16777216,171966464~16777216,197132288~12582912,213909504~71303168,289406976~4194304,306184192~20971520,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.484 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 13 diff [4194304~20971520,41943040~4194304,50331648~54525952,125829120~8388608,146800640~20971520,171966464~12582912,197132288~8388608,213909504~4194304,222298112~8388608,234881024~50331648,289406976~4194304,306184192~16777216,327155712~8388608,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.484 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,125829120~8388608,146800640~20971520,171966464~12582912,197132288~8388608,213909504~4194304,222298112~8388608,234881024~50331648,289406976~4194304,306184192~16777216,327155712~8388608,348127232~20971520,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.493 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,125829120~12582912,146800640~20971520,171966464~16777216,197132288~12582912,213909504~71303168,289406976~4194304,306184192~29360128,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.493 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 14 diff [4194304~20971520,41943040~4194304,50331648~54525952,121634816~16777216,146800640~20971520,171966464~12582912,197132288~8388608,213909504~16777216,234881024~50331648,289406976~4194304,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.493 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,121634816~16777216,146800640~20971520,171966464~12582912,197132288~8388608,213909504~16777216,234881024~50331648,289406976~4194304,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.503 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,121634816~16777216,146800640~20971520,171966464~16777216,197132288~12582912,213909504~71303168,289406976~4194304,306184192~67108864,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.503 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 15 diff [4194304~20971520,41943040~4194304,50331648~54525952,117440512~20971520,146800640~20971520,171966464~12582912,197132288~8388608,213909504~16777216,234881024~58720256,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.503 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,117440512~20971520,146800640~20971520,171966464~12582912,197132288~8388608,213909504~16777216,234881024~58720256,306184192~16777216,327155712~16777216,348127232~25165824,377487360~12582912,394264576~16777216] 2024-03-07T03:50:23.513 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,117440512~20971520,146800640~20971520,171966464~16777216,197132288~12582912,213909504~79691776,306184192~83886080,394264576~16777216] 2024-03-07T03:50:23.514 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 16 diff [4194304~20971520,41943040~4194304,50331648~54525952,117440512~50331648,171966464~12582912,197132288~8388608,213909504~16777216,234881024~62914560,306184192~16777216,327155712~46137344,377487360~33554432] 2024-03-07T03:50:23.514 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~54525952,117440512~50331648,171966464~12582912,197132288~8388608,213909504~16777216,234881024~62914560,306184192~16777216,327155712~46137344,377487360~33554432] 2024-03-07T03:50:23.525 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,117440512~50331648,171966464~16777216,197132288~12582912,213909504~83886080,306184192~104857600] 2024-03-07T03:50:23.525 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 17 diff [4194304~25165824,41943040~4194304,50331648~54525952,109051904~58720256,171966464~12582912,197132288~8388608,213909504~16777216,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.525 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~4194304,50331648~54525952,109051904~58720256,171966464~12582912,197132288~8388608,213909504~16777216,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.537 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~117440512,171966464~16777216,197132288~12582912,213909504~83886080,306184192~104857600] 2024-03-07T03:50:23.537 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 18 diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~12582912,197132288~8388608,213909504~16777216,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.537 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~12582912,197132288~8388608,213909504~16777216,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.549 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~12582912,213909504~83886080,306184192~104857600] 2024-03-07T03:50:23.550 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to 19 diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.550 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:23.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:23 smithi148 bash[79819]: cluster 2024-03-07T03:50:22.168709+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2024-03-07T03:50:23.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:23 smithi148 bash[82222]: cluster 2024-03-07T03:50:22.168709+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2024-03-07T03:50:23.563 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:23.563 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 9 diff [4194304~12582912,50331648~4194304,75497472~8388608,88080384~8388608,171966464~8388608,251658240~4194304,260046848~8388608,276824064~8388608,352321536~8388608] 2024-03-07T03:50:23.563 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,50331648~4194304,75497472~8388608,88080384~8388608,171966464~8388608,251658240~4194304,260046848~8388608,276824064~8388608,352321536~8388608] 2024-03-07T03:50:23.567 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,50331648~8388608,75497472~8388608,88080384~12582912,171966464~8388608,251658240~4194304,260046848~12582912,276824064~8388608,352321536~12582912] 2024-03-07T03:50:23.568 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 10 diff [4194304~12582912,41943040~4194304,50331648~4194304,75497472~8388608,88080384~8388608,171966464~8388608,213909504~4194304,222298112~4194304,234881024~12582912,251658240~4194304,260046848~8388608,276824064~8388608,306184192~12582912,352321536~8388608,385875968~4194304,394264576~8388608] 2024-03-07T03:50:23.568 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~12582912,41943040~4194304,50331648~4194304,75497472~8388608,88080384~8388608,171966464~8388608,213909504~4194304,222298112~4194304,234881024~12582912,251658240~4194304,260046848~8388608,276824064~8388608,306184192~12582912,352321536~8388608,385875968~4194304,394264576~8388608] 2024-03-07T03:50:23.571 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~16777216,41943040~4194304,50331648~8388608,75497472~8388608,88080384~12582912,171966464~8388608,213909504~16777216,234881024~20971520,260046848~12582912,276824064~8388608,306184192~16777216,352321536~12582912,385875968~4194304,394264576~12582912] 2024-03-07T03:50:23.571 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 11 diff [4194304~20971520,41943040~4194304,50331648~4194304,71303168~25165824,100663296~4194304,171966464~12582912,213909504~4194304,222298112~8388608,234881024~12582912,251658240~4194304,260046848~25165824,306184192~16777216,352321536~8388608,381681664~8388608,394264576~8388608] 2024-03-07T03:50:23.571 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~4194304,71303168~25165824,100663296~4194304,171966464~12582912,213909504~4194304,222298112~8388608,234881024~12582912,251658240~4194304,260046848~25165824,306184192~16777216,352321536~8388608,381681664~8388608,394264576~8388608] 2024-03-07T03:50:23.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:23 smithi195 bash[66542]: cluster 2024-03-07T03:50:22.168709+0000 mon.a (mon.0) 1933 : cluster 0 osdmap e1077: 8 total, 8 up, 8 in 2024-03-07T03:50:23.576 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~8388608,71303168~37748736,171966464~16777216,213909504~41943040,260046848~25165824,306184192~20971520,352321536~12582912,381681664~8388608,394264576~12582912] 2024-03-07T03:50:23.576 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 12 diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~37748736,100663296~4194304,125829120~8388608,146800640~8388608,171966464~12582912,213909504~4194304,222298112~8388608,234881024~12582912,251658240~33554432,289406976~4194304,306184192~16777216,352321536~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.576 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~37748736,100663296~4194304,125829120~8388608,146800640~8388608,171966464~12582912,213909504~4194304,222298112~8388608,234881024~12582912,251658240~33554432,289406976~4194304,306184192~16777216,352321536~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.582 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,125829120~12582912,146800640~12582912,171966464~16777216,213909504~71303168,289406976~4194304,306184192~20971520,352321536~12582912,377487360~12582912,394264576~12582912] 2024-03-07T03:50:23.583 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 13 diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,125829120~8388608,146800640~8388608,159383552~8388608,171966464~12582912,213909504~4194304,222298112~8388608,234881024~50331648,289406976~4194304,306184192~16777216,327155712~8388608,352321536~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.583 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,125829120~8388608,146800640~8388608,159383552~8388608,171966464~12582912,213909504~4194304,222298112~8388608,234881024~50331648,289406976~4194304,306184192~16777216,327155712~8388608,352321536~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.590 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,125829120~12582912,146800640~20971520,171966464~16777216,213909504~71303168,289406976~4194304,306184192~29360128,352321536~12582912,377487360~12582912,394264576~12582912] 2024-03-07T03:50:23.590 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 14 diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,121634816~16777216,146800640~8388608,159383552~8388608,171966464~12582912,213909504~16777216,234881024~50331648,289406976~4194304,306184192~16777216,327155712~16777216,352321536~8388608,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.590 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,121634816~16777216,146800640~8388608,159383552~8388608,171966464~12582912,213909504~16777216,234881024~50331648,289406976~4194304,306184192~16777216,327155712~16777216,352321536~8388608,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.598 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,121634816~16777216,146800640~20971520,171966464~16777216,213909504~71303168,289406976~4194304,306184192~41943040,352321536~20971520,377487360~12582912,394264576~12582912] 2024-03-07T03:50:23.598 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 15 diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,117440512~20971520,146800640~8388608,159383552~8388608,171966464~12582912,213909504~16777216,234881024~58720256,306184192~16777216,327155712~16777216,352321536~8388608,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.598 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,117440512~20971520,146800640~8388608,159383552~8388608,171966464~12582912,213909504~16777216,234881024~58720256,306184192~16777216,327155712~16777216,352321536~8388608,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.608 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,117440512~20971520,146800640~20971520,171966464~16777216,213909504~79691776,306184192~41943040,352321536~37748736,394264576~12582912] 2024-03-07T03:50:23.608 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 16 diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,117440512~37748736,159383552~8388608,171966464~12582912,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~8388608,377487360~25165824] 2024-03-07T03:50:23.608 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~20971520,41943040~4194304,50331648~4194304,58720256~46137344,117440512~37748736,159383552~8388608,171966464~12582912,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~8388608,377487360~25165824] 2024-03-07T03:50:23.618 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~58720256,117440512~50331648,171966464~16777216,213909504~83886080,306184192~41943040,352321536~54525952] 2024-03-07T03:50:23.618 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 17 diff [4194304~25165824,41943040~4194304,50331648~4194304,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:23.618 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~4194304,50331648~4194304,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:23.628 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~4194304,50331648~117440512,171966464~16777216,201326592~4194304,213909504~83886080,306184192~41943040,352321536~54525952] 2024-03-07T03:50:23.629 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 18 diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:23.629 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:23.640 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,213909504~83886080,306184192~41943040,352321536~54525952] 2024-03-07T03:50:23.640 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to 19 diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:23.641 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:23.654 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:23.654 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 10 diff [41943040~4194304,79691776~4194304,213909504~4194304,222298112~4194304,234881024~12582912,306184192~12582912,385875968~4194304,394264576~8388608] 2024-03-07T03:50:23.655 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [41943040~4194304,79691776~4194304,213909504~4194304,222298112~4194304,234881024~12582912,306184192~12582912,385875968~4194304,394264576~8388608] 2024-03-07T03:50:23.658 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [41943040~4194304,79691776~4194304,213909504~16777216,234881024~16777216,306184192~16777216,385875968~4194304,394264576~12582912] 2024-03-07T03:50:23.658 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 11 diff [16777216~8388608,41943040~4194304,71303168~4194304,79691776~8388608,100663296~4194304,180355072~4194304,213909504~4194304,222298112~8388608,234881024~12582912,268435456~8388608,306184192~16777216,381681664~8388608,394264576~8388608] 2024-03-07T03:50:23.658 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [16777216~8388608,41943040~4194304,71303168~4194304,79691776~8388608,100663296~4194304,180355072~4194304,213909504~4194304,222298112~8388608,234881024~12582912,268435456~8388608,306184192~16777216,381681664~8388608,394264576~8388608] 2024-03-07T03:50:23.662 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [16777216~12582912,41943040~4194304,71303168~16777216,100663296~8388608,180355072~8388608,213909504~37748736,268435456~8388608,306184192~20971520,381681664~8388608,394264576~12582912] 2024-03-07T03:50:23.662 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 12 diff [16777216~8388608,41943040~4194304,58720256~16777216,79691776~8388608,100663296~4194304,125829120~8388608,146800640~8388608,171966464~4194304,180355072~4194304,213909504~4194304,222298112~8388608,234881024~12582912,255852544~4194304,268435456~8388608,289406976~4194304,306184192~16777216,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.662 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [16777216~8388608,41943040~4194304,58720256~16777216,79691776~8388608,100663296~4194304,125829120~8388608,146800640~8388608,171966464~4194304,180355072~4194304,213909504~4194304,222298112~8388608,234881024~12582912,255852544~4194304,268435456~8388608,289406976~4194304,306184192~16777216,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.668 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [16777216~12582912,41943040~4194304,58720256~29360128,100663296~8388608,125829120~12582912,146800640~12582912,171966464~16777216,213909504~37748736,255852544~8388608,268435456~8388608,289406976~4194304,306184192~20971520,377487360~12582912,394264576~12582912] 2024-03-07T03:50:23.668 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 13 diff [16777216~8388608,41943040~4194304,58720256~16777216,79691776~8388608,96468992~8388608,125829120~8388608,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~4194304,222298112~8388608,234881024~25165824,268435456~8388608,289406976~4194304,306184192~16777216,327155712~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.668 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [16777216~8388608,41943040~4194304,58720256~16777216,79691776~8388608,96468992~8388608,125829120~8388608,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~4194304,222298112~8388608,234881024~25165824,268435456~8388608,289406976~4194304,306184192~16777216,327155712~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.673 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [16777216~12582912,41943040~4194304,58720256~29360128,96468992~12582912,125829120~12582912,146800640~20971520,171966464~16777216,213909504~50331648,268435456~8388608,289406976~4194304,306184192~29360128,377487360~12582912,394264576~12582912] 2024-03-07T03:50:23.674 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 14 diff [12582912~12582912,41943040~4194304,58720256~29360128,96468992~8388608,121634816~16777216,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,234881024~41943040,289406976~4194304,306184192~16777216,327155712~16777216,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.674 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,41943040~4194304,58720256~29360128,96468992~8388608,121634816~16777216,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,234881024~41943040,289406976~4194304,306184192~16777216,327155712~16777216,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.680 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~16777216,41943040~4194304,58720256~29360128,96468992~12582912,121634816~16777216,146800640~20971520,171966464~16777216,213909504~62914560,289406976~4194304,306184192~41943040,364904448~8388608,377487360~12582912,394264576~12582912] 2024-03-07T03:50:23.681 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 15 diff [12582912~16777216,41943040~4194304,58720256~29360128,96468992~8388608,117440512~20971520,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,234881024~41943040,281018368~12582912,306184192~16777216,327155712~16777216,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.681 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~16777216,41943040~4194304,58720256~29360128,96468992~8388608,117440512~20971520,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,234881024~41943040,281018368~12582912,306184192~16777216,327155712~16777216,364904448~8388608,377487360~12582912,394264576~8388608] 2024-03-07T03:50:23.689 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~16777216,41943040~4194304,58720256~29360128,96468992~12582912,117440512~20971520,146800640~20971520,171966464~16777216,213909504~62914560,281018368~12582912,306184192~41943040,364904448~25165824,394264576~12582912] 2024-03-07T03:50:23.689 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 16 diff [12582912~16777216,41943040~4194304,58720256~29360128,96468992~8388608,117440512~37748736,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,364904448~8388608,377487360~25165824] 2024-03-07T03:50:23.689 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~16777216,41943040~4194304,58720256~29360128,96468992~8388608,117440512~37748736,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,364904448~8388608,377487360~25165824] 2024-03-07T03:50:23.699 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~16777216,41943040~4194304,58720256~29360128,96468992~12582912,117440512~50331648,171966464~16777216,213909504~83886080,306184192~41943040,364904448~41943040] 2024-03-07T03:50:23.699 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 17 diff [4194304~4194304,12582912~16777216,41943040~4194304,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.699 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~4194304,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.709 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,12582912~16777216,41943040~4194304,58720256~33554432,96468992~71303168,171966464~16777216,201326592~4194304,213909504~83886080,306184192~41943040,364904448~41943040] 2024-03-07T03:50:23.709 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 18 diff [4194304~4194304,12582912~16777216,41943040~8388608,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.709 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~8388608,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.720 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~8388608,58720256~33554432,96468992~71303168,171966464~16777216,201326592~4194304,213909504~83886080,306184192~41943040,364904448~41943040] 2024-03-07T03:50:23.720 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to 19 diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.720 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.732 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:23.733 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 11 diff [16777216~8388608,71303168~4194304,83886080~4194304,100663296~4194304,180355072~4194304,222298112~8388608,268435456~8388608,318767104~8388608,381681664~4194304] 2024-03-07T03:50:23.733 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [16777216~8388608,71303168~4194304,83886080~4194304,100663296~4194304,180355072~4194304,222298112~8388608,268435456~8388608,318767104~8388608,381681664~4194304] 2024-03-07T03:50:23.736 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [16777216~12582912,71303168~8388608,83886080~4194304,100663296~8388608,180355072~8388608,222298112~12582912,268435456~8388608,318767104~8388608,381681664~4194304] 2024-03-07T03:50:23.736 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 12 diff [16777216~8388608,58720256~16777216,83886080~4194304,100663296~4194304,125829120~8388608,146800640~8388608,171966464~4194304,180355072~4194304,222298112~8388608,255852544~4194304,268435456~8388608,289406976~4194304,318767104~8388608,377487360~8388608] 2024-03-07T03:50:23.736 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [16777216~8388608,58720256~16777216,83886080~4194304,100663296~4194304,125829120~8388608,146800640~8388608,171966464~4194304,180355072~4194304,222298112~8388608,255852544~4194304,268435456~8388608,289406976~4194304,318767104~8388608,377487360~8388608] 2024-03-07T03:50:23.741 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [16777216~12582912,58720256~20971520,83886080~4194304,100663296~8388608,125829120~12582912,146800640~12582912,171966464~16777216,222298112~12582912,255852544~8388608,268435456~8388608,289406976~4194304,318767104~8388608,377487360~8388608] 2024-03-07T03:50:23.741 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 13 diff [16777216~8388608,58720256~16777216,83886080~4194304,96468992~8388608,125829120~8388608,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,222298112~8388608,239075328~8388608,251658240~8388608,268435456~8388608,289406976~4194304,318767104~16777216,377487360~8388608] 2024-03-07T03:50:23.741 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [16777216~8388608,58720256~16777216,83886080~4194304,96468992~8388608,125829120~8388608,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,222298112~8388608,239075328~8388608,251658240~8388608,268435456~8388608,289406976~4194304,318767104~16777216,377487360~8388608] 2024-03-07T03:50:23.746 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [16777216~12582912,58720256~20971520,83886080~4194304,96468992~12582912,125829120~12582912,146800640~20971520,171966464~16777216,222298112~12582912,239075328~25165824,268435456~8388608,289406976~4194304,318767104~16777216,377487360~8388608] 2024-03-07T03:50:23.746 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 14 diff [12582912~12582912,58720256~20971520,83886080~4194304,96468992~8388608,121634816~16777216,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~25165824,289406976~4194304,318767104~25165824,364904448~8388608,377487360~8388608,394264576~8388608] 2024-03-07T03:50:23.746 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,58720256~20971520,83886080~4194304,96468992~8388608,121634816~16777216,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~25165824,289406976~4194304,318767104~25165824,364904448~8388608,377487360~8388608,394264576~8388608] 2024-03-07T03:50:23.752 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~16777216,58720256~29360128,96468992~12582912,121634816~16777216,146800640~20971520,171966464~16777216,213909504~20971520,239075328~37748736,289406976~4194304,318767104~29360128,364904448~8388608,377487360~8388608,394264576~8388608] 2024-03-07T03:50:23.753 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 15 diff [12582912~16777216,58720256~29360128,96468992~8388608,117440512~20971520,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~25165824,281018368~12582912,318767104~25165824,364904448~8388608,377487360~8388608,394264576~8388608] 2024-03-07T03:50:23.753 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~16777216,58720256~29360128,96468992~8388608,117440512~20971520,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~25165824,281018368~12582912,318767104~25165824,364904448~8388608,377487360~8388608,394264576~8388608] 2024-03-07T03:50:23.759 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~16777216,58720256~29360128,96468992~12582912,117440512~20971520,146800640~20971520,171966464~16777216,213909504~20971520,239075328~37748736,281018368~12582912,318767104~29360128,364904448~20971520,394264576~8388608] 2024-03-07T03:50:23.760 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 16 diff [12582912~16777216,58720256~29360128,96468992~8388608,117440512~37748736,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~29360128,364904448~8388608,377487360~8388608,390070272~8388608] 2024-03-07T03:50:23.760 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~16777216,58720256~29360128,96468992~8388608,117440512~37748736,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~29360128,364904448~8388608,377487360~8388608,390070272~8388608] 2024-03-07T03:50:23.767 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~16777216,58720256~29360128,96468992~12582912,117440512~50331648,171966464~16777216,213909504~20971520,239075328~58720256,318767104~29360128,364904448~20971520,390070272~12582912] 2024-03-07T03:50:23.768 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 17 diff [4194304~4194304,12582912~16777216,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~29360128,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.768 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~29360128,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.777 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,12582912~16777216,58720256~33554432,96468992~71303168,171966464~16777216,201326592~4194304,213909504~20971520,239075328~58720256,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.777 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 18 diff [4194304~4194304,12582912~16777216,46137344~4194304,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:23.777 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,46137344~4194304,58720256~29360128,96468992~8388608,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:23.787 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,46137344~4194304,58720256~33554432,96468992~71303168,171966464~16777216,201326592~4194304,213909504~20971520,239075328~58720256,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.787 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to 19 diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:23.787 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:23.799 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.799 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 12 diff [58720256~12582912,125829120~8388608,146800640~8388608,171966464~4194304,180355072~4194304,255852544~4194304,289406976~4194304,377487360~4194304] 2024-03-07T03:50:23.799 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [58720256~12582912,125829120~8388608,146800640~8388608,171966464~4194304,180355072~4194304,255852544~4194304,289406976~4194304,377487360~4194304] 2024-03-07T03:50:23.802 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [58720256~12582912,125829120~12582912,146800640~12582912,171966464~16777216,255852544~8388608,289406976~4194304,377487360~8388608] 2024-03-07T03:50:23.802 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 13 diff [58720256~12582912,96468992~4194304,125829120~8388608,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,222298112~8388608,239075328~8388608,251658240~8388608,289406976~4194304,327155712~8388608,377487360~4194304] 2024-03-07T03:50:23.803 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [58720256~12582912,96468992~4194304,125829120~8388608,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,222298112~8388608,239075328~8388608,251658240~8388608,289406976~4194304,327155712~8388608,377487360~4194304] 2024-03-07T03:50:23.807 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [58720256~16777216,96468992~4194304,125829120~12582912,146800640~20971520,171966464~16777216,222298112~8388608,239075328~25165824,289406976~4194304,327155712~8388608,377487360~8388608] 2024-03-07T03:50:23.807 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 14 diff [12582912~8388608,58720256~12582912,75497472~8388608,96468992~4194304,121634816~16777216,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~16777216,289406976~4194304,318767104~4194304,327155712~16777216,364904448~8388608,377487360~4194304,394264576~8388608] 2024-03-07T03:50:23.807 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~8388608,58720256~12582912,75497472~8388608,96468992~4194304,121634816~16777216,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~16777216,289406976~4194304,318767104~4194304,327155712~16777216,364904448~8388608,377487360~4194304,394264576~8388608] 2024-03-07T03:50:23.811 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~8388608,58720256~25165824,96468992~4194304,121634816~16777216,146800640~20971520,171966464~16777216,213909504~16777216,239075328~33554432,289406976~4194304,318767104~29360128,364904448~8388608,377487360~8388608,394264576~8388608] 2024-03-07T03:50:23.811 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 15 diff [12582912~12582912,58720256~12582912,75497472~12582912,96468992~4194304,117440512~20971520,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~16777216,281018368~12582912,318767104~4194304,327155712~16777216,364904448~8388608,377487360~4194304,394264576~8388608] 2024-03-07T03:50:23.812 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,58720256~12582912,75497472~12582912,96468992~4194304,117440512~20971520,146800640~8388608,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~16777216,281018368~12582912,318767104~4194304,327155712~16777216,364904448~8388608,377487360~4194304,394264576~8388608] 2024-03-07T03:50:23.818 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~12582912,58720256~29360128,96468992~4194304,117440512~20971520,146800640~20971520,171966464~16777216,213909504~16777216,239075328~33554432,281018368~12582912,318767104~29360128,364904448~20971520,394264576~8388608] 2024-03-07T03:50:23.818 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 16 diff [12582912~12582912,58720256~12582912,75497472~12582912,96468992~4194304,117440512~37748736,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~16777216,272629760~25165824,318767104~4194304,327155712~20971520,364904448~8388608,377487360~4194304,390070272~8388608] 2024-03-07T03:50:23.818 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,58720256~12582912,75497472~12582912,96468992~4194304,117440512~37748736,159383552~8388608,171966464~4194304,180355072~4194304,213909504~16777216,239075328~8388608,251658240~16777216,272629760~25165824,318767104~4194304,327155712~20971520,364904448~8388608,377487360~4194304,390070272~8388608] 2024-03-07T03:50:23.824 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~12582912,58720256~29360128,96468992~4194304,117440512~50331648,171966464~16777216,213909504~16777216,239075328~58720256,318767104~29360128,364904448~20971520,390070272~12582912] 2024-03-07T03:50:23.825 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 17 diff [4194304~4194304,12582912~12582912,58720256~12582912,75497472~16777216,96468992~4194304,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~16777216,272629760~25165824,318767104~4194304,327155712~20971520,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.825 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,58720256~12582912,75497472~16777216,96468992~4194304,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~16777216,272629760~25165824,318767104~4194304,327155712~20971520,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.832 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,12582912~16777216,58720256~33554432,96468992~4194304,109051904~58720256,171966464~16777216,201326592~4194304,213909504~16777216,239075328~58720256,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.833 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 18 diff [4194304~4194304,12582912~12582912,46137344~4194304,58720256~33554432,96468992~4194304,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.833 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,46137344~4194304,58720256~33554432,96468992~4194304,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,213909504~16777216,239075328~8388608,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.842 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,46137344~4194304,58720256~33554432,96468992~4194304,109051904~58720256,171966464~16777216,201326592~4194304,213909504~16777216,239075328~58720256,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.842 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to 19 diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.842 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.852 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.852 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 13 diff [62914560~12582912,96468992~4194304,159383552~8388608,222298112~8388608,239075328~8388608,251658240~4194304,327155712~8388608] 2024-03-07T03:50:23.852 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [62914560~12582912,96468992~4194304,159383552~8388608,222298112~8388608,239075328~8388608,251658240~4194304,327155712~8388608] 2024-03-07T03:50:23.855 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [62914560~12582912,96468992~4194304,159383552~8388608,222298112~8388608,239075328~20971520,327155712~8388608] 2024-03-07T03:50:23.855 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 14 diff [12582912~8388608,62914560~20971520,96468992~4194304,121634816~8388608,159383552~8388608,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,318767104~4194304,327155712~16777216,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.855 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~8388608,62914560~20971520,96468992~4194304,121634816~8388608,159383552~8388608,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,318767104~4194304,327155712~16777216,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.859 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~8388608,62914560~20971520,96468992~4194304,121634816~12582912,159383552~8388608,213909504~16777216,239075328~33554432,318767104~29360128,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.860 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 15 diff [12582912~12582912,62914560~25165824,96468992~4194304,117440512~20971520,159383552~8388608,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,281018368~8388608,318767104~4194304,327155712~16777216,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.860 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,62914560~25165824,96468992~4194304,117440512~20971520,159383552~8388608,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,281018368~8388608,318767104~4194304,327155712~16777216,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.865 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~12582912,62914560~25165824,96468992~4194304,117440512~20971520,159383552~8388608,213909504~16777216,239075328~33554432,281018368~12582912,318767104~29360128,364904448~12582912,394264576~8388608] 2024-03-07T03:50:23.865 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 16 diff [12582912~12582912,62914560~25165824,96468992~4194304,117440512~29360128,159383552~8388608,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~20971520,364904448~8388608,390070272~8388608] 2024-03-07T03:50:23.865 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,62914560~25165824,96468992~4194304,117440512~29360128,159383552~8388608,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~20971520,364904448~8388608,390070272~8388608] 2024-03-07T03:50:23.871 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~12582912,62914560~25165824,96468992~4194304,117440512~29360128,159383552~8388608,213909504~16777216,239075328~58720256,318767104~29360128,364904448~12582912,390070272~12582912] 2024-03-07T03:50:23.871 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 17 diff [4194304~4194304,12582912~12582912,62914560~29360128,96468992~4194304,109051904~37748736,150994944~16777216,201326592~4194304,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~20971520,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.871 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,62914560~29360128,96468992~4194304,109051904~37748736,150994944~16777216,201326592~4194304,213909504~16777216,239075328~8388608,251658240~4194304,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~20971520,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.878 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,12582912~16777216,62914560~29360128,96468992~4194304,109051904~37748736,150994944~16777216,201326592~4194304,213909504~16777216,239075328~58720256,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.878 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 18 diff [4194304~4194304,12582912~12582912,46137344~4194304,62914560~29360128,96468992~4194304,109051904~37748736,150994944~16777216,201326592~4194304,213909504~16777216,239075328~8388608,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.878 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,46137344~4194304,62914560~29360128,96468992~4194304,109051904~37748736,150994944~16777216,201326592~4194304,213909504~16777216,239075328~8388608,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.886 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,46137344~4194304,62914560~29360128,96468992~4194304,109051904~37748736,150994944~16777216,201326592~4194304,213909504~16777216,239075328~58720256,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.886 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to 19 diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.886 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:23.895 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.895 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 14 diff [12582912~8388608,75497472~8388608,121634816~8388608,213909504~8388608,247463936~4194304,260046848~8388608,318767104~4194304,335544320~8388608,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.895 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~8388608,75497472~8388608,121634816~8388608,213909504~8388608,247463936~4194304,260046848~8388608,318767104~4194304,335544320~8388608,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.898 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~8388608,75497472~8388608,121634816~12582912,213909504~8388608,247463936~4194304,260046848~12582912,318767104~8388608,335544320~12582912,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.898 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 15 diff [12582912~12582912,75497472~12582912,117440512~20971520,213909504~8388608,247463936~4194304,260046848~8388608,281018368~8388608,318767104~4194304,335544320~8388608,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.898 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,75497472~12582912,117440512~20971520,213909504~8388608,247463936~4194304,260046848~8388608,281018368~8388608,318767104~4194304,335544320~8388608,364904448~8388608,394264576~8388608] 2024-03-07T03:50:23.903 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~12582912,75497472~12582912,117440512~20971520,213909504~8388608,247463936~4194304,260046848~12582912,281018368~12582912,318767104~8388608,335544320~12582912,364904448~12582912,394264576~8388608] 2024-03-07T03:50:23.903 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 16 diff [12582912~12582912,75497472~12582912,117440512~29360128,213909504~8388608,247463936~8388608,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~8388608,390070272~8388608] 2024-03-07T03:50:23.903 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~12582912,75497472~12582912,117440512~29360128,213909504~8388608,247463936~8388608,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~8388608,390070272~8388608] 2024-03-07T03:50:23.908 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~12582912,75497472~12582912,117440512~29360128,213909504~8388608,247463936~8388608,260046848~37748736,318767104~12582912,335544320~12582912,364904448~12582912,390070272~12582912] 2024-03-07T03:50:23.908 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 17 diff [4194304~4194304,12582912~12582912,75497472~16777216,109051904~37748736,150994944~8388608,201326592~4194304,213909504~8388608,247463936~8388608,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.908 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,75497472~16777216,109051904~37748736,150994944~8388608,201326592~4194304,213909504~8388608,247463936~8388608,260046848~8388608,272629760~16777216,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.913 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,12582912~16777216,75497472~16777216,109051904~37748736,150994944~12582912,201326592~4194304,213909504~8388608,247463936~8388608,260046848~37748736,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.913 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 18 diff [4194304~4194304,12582912~12582912,46137344~4194304,67108864~25165824,109051904~37748736,150994944~8388608,201326592~4194304,213909504~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.914 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,46137344~4194304,67108864~25165824,109051904~37748736,150994944~8388608,201326592~4194304,213909504~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.920 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,46137344~4194304,67108864~25165824,109051904~37748736,150994944~12582912,201326592~4194304,213909504~8388608,247463936~50331648,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.920 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to 19 diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.920 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.929 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~37748736,150994944~12582912,201326592~4194304,209715200~12582912,234881024~8388608,247463936~50331648,318767104~29360128,364904448~37748736] 2024-03-07T03:50:23.929 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 15 diff [12582912~4194304,20971520~4194304,75497472~8388608,117440512~12582912,134217728~4194304,281018368~8388608,339738624~8388608,369098752~8388608] 2024-03-07T03:50:23.929 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~4194304,20971520~4194304,75497472~8388608,117440512~12582912,134217728~4194304,281018368~8388608,339738624~8388608,369098752~8388608] 2024-03-07T03:50:23.932 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~4194304,20971520~4194304,75497472~12582912,117440512~12582912,134217728~4194304,281018368~12582912,339738624~8388608,369098752~8388608] 2024-03-07T03:50:23.932 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 16 diff [12582912~4194304,20971520~4194304,75497472~8388608,117440512~12582912,134217728~12582912,251658240~4194304,272629760~16777216,293601280~4194304,327155712~4194304,335544320~12582912,364904448~8388608,390070272~8388608] 2024-03-07T03:50:23.932 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [12582912~4194304,20971520~4194304,75497472~8388608,117440512~12582912,134217728~12582912,251658240~4194304,272629760~16777216,293601280~4194304,327155712~4194304,335544320~12582912,364904448~8388608,390070272~8388608] 2024-03-07T03:50:23.936 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [12582912~4194304,20971520~4194304,75497472~12582912,117440512~12582912,134217728~12582912,251658240~4194304,272629760~25165824,327155712~4194304,335544320~12582912,364904448~12582912,390070272~12582912] 2024-03-07T03:50:23.936 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 17 diff [4194304~4194304,12582912~4194304,20971520~4194304,75497472~12582912,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,251658240~4194304,272629760~16777216,293601280~4194304,327155712~4194304,335544320~12582912,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.936 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~4194304,20971520~4194304,75497472~12582912,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,251658240~4194304,272629760~16777216,293601280~4194304,327155712~4194304,335544320~12582912,364904448~20971520,390070272~8388608] 2024-03-07T03:50:23.942 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,12582912~4194304,20971520~8388608,75497472~16777216,109051904~20971520,134217728~12582912,150994944~12582912,201326592~4194304,251658240~4194304,272629760~25165824,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.942 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 18 diff [4194304~4194304,12582912~12582912,46137344~4194304,67108864~20971520,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.942 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,46137344~4194304,67108864~20971520,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.947 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,46137344~4194304,67108864~25165824,109051904~20971520,134217728~12582912,150994944~12582912,201326592~4194304,251658240~46137344,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.947 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to 19 diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.947 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:23.955 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~20971520,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~46137344,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.955 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 16 diff [134217728~12582912,251658240~4194304,272629760~8388608,293601280~4194304,327155712~4194304,335544320~4194304,364904448~4194304,390070272~8388608] 2024-03-07T03:50:23.955 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [134217728~12582912,251658240~4194304,272629760~8388608,293601280~4194304,327155712~4194304,335544320~4194304,364904448~4194304,390070272~8388608] 2024-03-07T03:50:23.958 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [134217728~12582912,251658240~4194304,272629760~8388608,293601280~4194304,327155712~4194304,335544320~8388608,364904448~8388608,390070272~12582912] 2024-03-07T03:50:23.958 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 17 diff [4194304~4194304,20971520~4194304,83886080~4194304,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,251658240~4194304,272629760~8388608,293601280~4194304,327155712~4194304,335544320~4194304,364904448~4194304,373293056~12582912,390070272~8388608] 2024-03-07T03:50:23.958 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,20971520~4194304,83886080~4194304,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,251658240~4194304,272629760~8388608,293601280~4194304,327155712~4194304,335544320~4194304,364904448~4194304,373293056~12582912,390070272~8388608] 2024-03-07T03:50:23.962 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,20971520~8388608,83886080~8388608,109051904~8388608,134217728~12582912,150994944~12582912,201326592~4194304,251658240~4194304,272629760~8388608,293601280~4194304,327155712~16777216,364904448~37748736] 2024-03-07T03:50:23.962 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 18 diff [4194304~4194304,16777216~8388608,46137344~4194304,67108864~8388608,83886080~4194304,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:23.962 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,46137344~4194304,67108864~8388608,83886080~4194304,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:23.968 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,46137344~4194304,67108864~12582912,83886080~8388608,109051904~12582912,134217728~12582912,150994944~12582912,201326592~4194304,251658240~29360128,293601280~4194304,327155712~16777216,364904448~37748736] 2024-03-07T03:50:23.968 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to 19 diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:23.968 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:23.975 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,67108864~12582912,83886080~16777216,109051904~12582912,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~29360128,293601280~4194304,327155712~20971520,364904448~37748736] 2024-03-07T03:50:23.975 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to 17 diff [4194304~4194304,20971520~4194304,83886080~4194304,109051904~8388608,150994944~8388608,201326592~4194304,327155712~4194304,373293056~12582912] 2024-03-07T03:50:23.975 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,20971520~4194304,83886080~4194304,109051904~8388608,150994944~8388608,201326592~4194304,327155712~4194304,373293056~12582912] 2024-03-07T03:50:23.978 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~4194304,20971520~8388608,83886080~8388608,109051904~8388608,150994944~12582912,201326592~4194304,327155712~8388608,373293056~16777216] 2024-03-07T03:50:23.978 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to 18 diff [4194304~4194304,16777216~8388608,46137344~4194304,67108864~8388608,83886080~4194304,109051904~8388608,150994944~8388608,201326592~4194304,251658240~4194304,260046848~16777216,327155712~4194304,373293056~20971520] 2024-03-07T03:50:23.978 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,46137344~4194304,67108864~8388608,83886080~4194304,109051904~8388608,150994944~8388608,201326592~4194304,251658240~4194304,260046848~16777216,327155712~4194304,373293056~20971520] 2024-03-07T03:50:23.982 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,46137344~4194304,67108864~12582912,83886080~8388608,109051904~12582912,150994944~12582912,201326592~4194304,251658240~25165824,327155712~8388608,373293056~20971520] 2024-03-07T03:50:23.982 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to 19 diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~25165824,327155712~4194304,343932928~4194304,373293056~20971520] 2024-03-07T03:50:23.982 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~25165824,327155712~4194304,343932928~4194304,373293056~20971520] 2024-03-07T03:50:23.988 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,67108864~12582912,83886080~16777216,109051904~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~29360128,327155712~8388608,343932928~4194304,373293056~20971520] 2024-03-07T03:50:23.988 INFO:tasks.workunit.client.0.smithi148.stdout:from 17 to 18 diff [4194304~4194304,16777216~4194304,46137344~4194304,67108864~8388608,113246208~4194304,251658240~4194304,260046848~16777216,377487360~4194304,385875968~8388608] 2024-03-07T03:50:23.988 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~4194304,46137344~4194304,67108864~8388608,113246208~4194304,251658240~4194304,260046848~16777216,377487360~4194304,385875968~8388608] 2024-03-07T03:50:23.992 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~4194304,46137344~4194304,67108864~12582912,113246208~8388608,251658240~25165824,377487360~16777216] 2024-03-07T03:50:23.992 INFO:tasks.workunit.client.0.smithi148.stdout:from 17 to 19 diff [4194304~4194304,16777216~4194304,41943040~12582912,67108864~8388608,92274688~8388608,113246208~4194304,150994944~8388608,209715200~4194304,234881024~8388608,251658240~25165824,343932928~4194304,377487360~4194304,385875968~8388608] 2024-03-07T03:50:23.992 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~4194304,41943040~12582912,67108864~8388608,92274688~8388608,113246208~4194304,150994944~8388608,209715200~4194304,234881024~8388608,251658240~25165824,343932928~4194304,377487360~4194304,385875968~8388608] 2024-03-07T03:50:23.997 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~4194304,41943040~16777216,67108864~12582912,92274688~8388608,113246208~8388608,150994944~8388608,209715200~8388608,234881024~8388608,251658240~29360128,343932928~4194304,377487360~16777216] 2024-03-07T03:50:23.997 INFO:tasks.workunit.client.0.smithi148.stdout:from 18 to 19 diff [41943040~12582912,92274688~8388608,150994944~8388608,209715200~4194304,234881024~8388608,255852544~4194304,268435456~8388608,343932928~4194304] 2024-03-07T03:50:23.998 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [41943040~12582912,92274688~8388608,150994944~8388608,209715200~4194304,234881024~8388608,255852544~4194304,268435456~8388608,343932928~4194304] 2024-03-07T03:50:24.000 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [41943040~16777216,92274688~8388608,150994944~8388608,209715200~8388608,234881024~8388608,255852544~8388608,268435456~12582912,343932928~4194304] 2024-03-07T03:50:24.194 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:24.194 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:24.237 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:24.237 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:24.237 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:24.256 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:24.256 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:24.256 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:24.274 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:24.274 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:24.274 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:24.292 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:24.292 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:24.292 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:24.308 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:24.309 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:24.309 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:24.324 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:24.324 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:24.324 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:24.339 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:24.339 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:24.339 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:24.353 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:24.353 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:24.353 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:24.365 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:24.365 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:24.365 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:24.380 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:24.380 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:24.380 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:24.387 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:24.387 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:24.387 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:24.396 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:24.397 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:24.397 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:24.405 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:24.405 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:24.406 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:24.412 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~37748736,150994944~12582912,201326592~4194304,209715200~12582912,234881024~8388608,247463936~50331648,318767104~29360128,364904448~37748736] 2024-03-07T03:50:24.412 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:24.412 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:24.418 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~20971520,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~46137344,327155712~20971520,364904448~37748736] 2024-03-07T03:50:24.418 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to HEAD diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:24.418 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:24.423 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,67108864~12582912,83886080~16777216,109051904~12582912,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~29360128,293601280~4194304,327155712~20971520,364904448~37748736] 2024-03-07T03:50:24.423 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to HEAD diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~25165824,327155712~4194304,343932928~4194304,373293056~20971520] 2024-03-07T03:50:24.423 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~25165824,327155712~4194304,343932928~4194304,373293056~20971520] 2024-03-07T03:50:24.427 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,67108864~12582912,83886080~16777216,109051904~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~29360128,327155712~8388608,343932928~4194304,373293056~20971520] 2024-03-07T03:50:24.427 INFO:tasks.workunit.client.0.smithi148.stdout:from 17 to HEAD diff [4194304~4194304,16777216~4194304,41943040~12582912,67108864~8388608,92274688~8388608,113246208~4194304,150994944~8388608,209715200~4194304,234881024~8388608,251658240~25165824,343932928~4194304,377487360~4194304,385875968~8388608] 2024-03-07T03:50:24.427 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~4194304,41943040~12582912,67108864~8388608,92274688~8388608,113246208~4194304,150994944~8388608,209715200~4194304,234881024~8388608,251658240~25165824,343932928~4194304,377487360~4194304,385875968~8388608] 2024-03-07T03:50:24.430 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~4194304,41943040~16777216,67108864~12582912,92274688~8388608,113246208~8388608,150994944~8388608,209715200~8388608,234881024~8388608,251658240~29360128,343932928~4194304,377487360~16777216] 2024-03-07T03:50:24.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:24 smithi148 bash[79819]: cluster 2024-03-07T03:50:23.261779+0000 mgr.y (mgr.34101) 1093 : cluster 0 pgmap v1789: 105 pgs: 105 active+clean; 815 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 27 MiB/s wr, 331 op/s 2024-03-07T03:50:24.550 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:24 smithi148 bash[82222]: cluster 2024-03-07T03:50:23.261779+0000 mgr.y (mgr.34101) 1093 : cluster 0 pgmap v1789: 105 pgs: 105 active+clean; 815 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 27 MiB/s wr, 331 op/s 2024-03-07T03:50:24.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:24 smithi195 bash[66542]: cluster 2024-03-07T03:50:23.261779+0000 mgr.y (mgr.34101) 1093 : cluster 0 pgmap v1789: 105 pgs: 105 active+clean; 815 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 201 KiB/s rd, 27 MiB/s wr, 331 op/s 2024-03-07T03:50:25.223 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:25.223 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:25.253 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:25.253 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:25.253 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:25.270 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:25.270 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:25.270 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:25.285 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:25.286 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:25.286 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:25.301 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:25.301 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:25.301 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:25.315 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:25.315 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:25.315 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:25.328 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:25.328 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:25.329 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:25.341 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:25.341 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:25.341 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:25.353 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:25.353 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:25.353 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:25.364 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:25.364 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:25.364 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:25.373 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:25.373 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:25.373 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:25.382 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:25.382 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:25.382 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:25.390 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:25.390 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:25.390 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:25.397 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:25.397 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:25.397 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:25.403 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~37748736,150994944~12582912,201326592~4194304,209715200~12582912,234881024~8388608,247463936~50331648,318767104~29360128,364904448~37748736] 2024-03-07T03:50:25.403 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:25.403 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:25.408 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~20971520,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~46137344,327155712~20971520,364904448~37748736] 2024-03-07T03:50:25.408 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to HEAD diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:25.408 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:25.413 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,67108864~12582912,83886080~16777216,109051904~12582912,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~29360128,293601280~4194304,327155712~20971520,364904448~37748736] 2024-03-07T03:50:25.413 INFO:tasks.workunit.client.0.smithi148.stdout:from 16 to HEAD diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~25165824,327155712~4194304,343932928~4194304,373293056~20971520] 2024-03-07T03:50:25.413 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~25165824,327155712~4194304,343932928~4194304,373293056~20971520] 2024-03-07T03:50:25.415 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,67108864~12582912,83886080~16777216,109051904~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~29360128,327155712~8388608,343932928~4194304,373293056~20971520] 2024-03-07T03:50:25.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:25 smithi148 bash[79819]: cluster 2024-03-07T03:50:24.179311+0000 mon.a (mon.0) 1934 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2024-03-07T03:50:25.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:25 smithi148 bash[82222]: cluster 2024-03-07T03:50:24.179311+0000 mon.a (mon.0) 1934 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2024-03-07T03:50:25.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:25 smithi195 bash[66542]: cluster 2024-03-07T03:50:24.179311+0000 mon.a (mon.0) 1934 : cluster 0 osdmap e1078: 8 total, 8 up, 8 in 2024-03-07T03:50:26.229 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:26.229 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:26.267 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:26.267 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:26.267 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:26.280 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:26.281 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:26.281 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:26.292 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:26.292 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:26.293 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:26.305 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:26.305 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:26.305 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:26.316 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:26.317 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:26.317 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:26.328 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:26.328 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:26.328 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:26.339 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:26.339 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:26.339 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:26.349 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:26.349 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:26.349 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:26.357 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:26.357 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:26.357 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:26.365 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:26.366 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:26.366 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:26.373 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:26.373 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:26.373 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:26.378 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:26.378 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:26.378 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:26.384 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:26.384 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:26.384 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:26.388 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~37748736,150994944~12582912,201326592~4194304,209715200~12582912,234881024~8388608,247463936~50331648,318767104~29360128,364904448~37748736] 2024-03-07T03:50:26.389 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:26.389 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:26.392 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~20971520,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~46137344,327155712~20971520,364904448~37748736] 2024-03-07T03:50:26.392 INFO:tasks.workunit.client.0.smithi148.stdout:from 15 to HEAD diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:26.392 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,16777216~8388608,41943040~12582912,67108864~8388608,83886080~4194304,92274688~8388608,109051904~8388608,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~29360128,293601280~4194304,327155712~4194304,335544320~4194304,343932928~4194304,364904448~4194304,373293056~25165824] 2024-03-07T03:50:26.395 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~8388608,16777216~12582912,41943040~16777216,67108864~12582912,83886080~16777216,109051904~12582912,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~29360128,293601280~4194304,327155712~20971520,364904448~37748736] 2024-03-07T03:50:26.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:26 smithi148 bash[79819]: cluster 2024-03-07T03:50:25.188129+0000 mon.a (mon.0) 1935 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2024-03-07T03:50:26.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:26 smithi148 bash[79819]: cluster 2024-03-07T03:50:25.262586+0000 mgr.y (mgr.34101) 1094 : cluster 0 pgmap v1792: 105 pgs: 105 active+clean; 861 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 499 KiB/s rd, 26 MiB/s wr, 898 op/s 2024-03-07T03:50:26.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:26 smithi148 bash[82222]: cluster 2024-03-07T03:50:25.188129+0000 mon.a (mon.0) 1935 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2024-03-07T03:50:26.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:26 smithi148 bash[82222]: cluster 2024-03-07T03:50:25.262586+0000 mgr.y (mgr.34101) 1094 : cluster 0 pgmap v1792: 105 pgs: 105 active+clean; 861 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 499 KiB/s rd, 26 MiB/s wr, 898 op/s 2024-03-07T03:50:26.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:26 smithi195 bash[66542]: cluster 2024-03-07T03:50:25.188129+0000 mon.a (mon.0) 1935 : cluster 0 osdmap e1079: 8 total, 8 up, 8 in 2024-03-07T03:50:26.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:26 smithi195 bash[66542]: cluster 2024-03-07T03:50:25.262586+0000 mgr.y (mgr.34101) 1094 : cluster 0 pgmap v1792: 105 pgs: 105 active+clean; 861 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 499 KiB/s rd, 26 MiB/s wr, 898 op/s 2024-03-07T03:50:27.234 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:27.234 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:27.276 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:27.276 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:27.276 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:27.288 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:27.288 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:27.288 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:27.300 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:27.300 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:27.300 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:27.310 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:27.311 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:27.311 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:27.321 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:27.321 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:27.321 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:27.331 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:27.331 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:27.331 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:27.340 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:27.340 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:27.340 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:27.349 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:27.349 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:27.349 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:27.357 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:27.357 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:27.357 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:27.365 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:27.365 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:27.365 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:27.370 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:27.370 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:27.371 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:27.375 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:27.375 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:27.375 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:27.380 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:27.380 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:27.380 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:27.383 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~37748736,150994944~12582912,201326592~4194304,209715200~12582912,234881024~8388608,247463936~50331648,318767104~29360128,364904448~37748736] 2024-03-07T03:50:27.383 INFO:tasks.workunit.client.0.smithi148.stdout:from 14 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:27.384 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~20971520,92274688~8388608,109051904~20971520,134217728~12582912,150994944~8388608,201326592~4194304,209715200~4194304,234881024~8388608,251658240~37748736,293601280~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:27.386 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~20971520,134217728~12582912,150994944~12582912,201326592~4194304,209715200~8388608,234881024~8388608,251658240~46137344,327155712~20971520,364904448~37748736] 2024-03-07T03:50:27.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:27 smithi148 bash[82222]: cluster 2024-03-07T03:50:26.199202+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2024-03-07T03:50:27.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:27 smithi148 bash[79819]: cluster 2024-03-07T03:50:26.199202+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2024-03-07T03:50:27.551 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:50:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:50:27] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:50:27.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:27 smithi195 bash[66542]: cluster 2024-03-07T03:50:26.199202+0000 mon.a (mon.0) 1936 : cluster 0 osdmap e1080: 8 total, 8 up, 8 in 2024-03-07T03:50:28.253 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:28.253 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:28.302 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:28.302 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:28.302 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:28.312 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:28.312 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:28.312 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:28.322 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:28.322 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:28.322 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:28.332 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:28.332 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:28.332 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:28.341 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:28.341 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:28.341 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:28.351 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:28.351 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:28.351 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:28.357 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:28.357 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:28.357 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:28.365 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:28.365 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:28.365 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:28.371 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:28.371 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:28.372 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:28.378 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:28.378 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:28.378 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:28.384 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:28.384 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:28.384 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:28.389 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:28.389 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:28.389 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:28.392 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:28.392 INFO:tasks.workunit.client.0.smithi148.stdout:from 13 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:28.392 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,67108864~33554432,109051904~37748736,150994944~8388608,201326592~4194304,209715200~8388608,234881024~8388608,247463936~41943040,293601280~4194304,318767104~4194304,327155712~4194304,335544320~12582912,364904448~33554432] 2024-03-07T03:50:28.395 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,67108864~33554432,109051904~37748736,150994944~12582912,201326592~4194304,209715200~12582912,234881024~8388608,247463936~50331648,318767104~29360128,364904448~37748736] 2024-03-07T03:50:28.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:28 smithi148 bash[79819]: cluster 2024-03-07T03:50:27.203919+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2024-03-07T03:50:28.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:28 smithi148 bash[79819]: cluster 2024-03-07T03:50:27.263134+0000 mgr.y (mgr.34101) 1095 : cluster 0 pgmap v1795: 105 pgs: 105 active+clean; 861 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 534 KiB/s rd, 11 MiB/s wr, 968 op/s 2024-03-07T03:50:28.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:28 smithi148 bash[79819]: audit 2024-03-07T03:50:27.353289+0000 mon.a (mon.0) 1938 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:28.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:28 smithi148 bash[82222]: cluster 2024-03-07T03:50:27.203919+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2024-03-07T03:50:28.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:28 smithi148 bash[82222]: cluster 2024-03-07T03:50:27.263134+0000 mgr.y (mgr.34101) 1095 : cluster 0 pgmap v1795: 105 pgs: 105 active+clean; 861 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 534 KiB/s rd, 11 MiB/s wr, 968 op/s 2024-03-07T03:50:28.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:28 smithi148 bash[82222]: audit 2024-03-07T03:50:27.353289+0000 mon.a (mon.0) 1938 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:28.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:28 smithi195 bash[66542]: cluster 2024-03-07T03:50:27.203919+0000 mon.a (mon.0) 1937 : cluster 0 osdmap e1081: 8 total, 8 up, 8 in 2024-03-07T03:50:28.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:28 smithi195 bash[66542]: cluster 2024-03-07T03:50:27.263134+0000 mgr.y (mgr.34101) 1095 : cluster 0 pgmap v1795: 105 pgs: 105 active+clean; 861 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 534 KiB/s rd, 11 MiB/s wr, 968 op/s 2024-03-07T03:50:28.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:28 smithi195 bash[66542]: audit 2024-03-07T03:50:27.353289+0000 mon.a (mon.0) 1938 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:29.278 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:29.278 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:29.311 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:29.311 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:29.312 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:29.320 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:29.320 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:29.321 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:29.329 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:29.329 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:29.329 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:29.337 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:29.337 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:29.337 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:29.344 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:29.344 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:29.344 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:29.352 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:29.352 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:29.352 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:29.358 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:29.358 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:29.358 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:29.364 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:29.364 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:29.365 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:29.370 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:29.370 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:29.370 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:29.374 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:29.374 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:29.374 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:29.378 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:29.378 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:29.378 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:29.381 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:29.381 INFO:tasks.workunit.client.0.smithi148.stdout:from 12 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:29.381 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~37748736,293601280~4194304,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:29.382 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~16777216,62914560~37748736,109051904~37748736,150994944~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:29.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:29 smithi148 bash[79819]: cluster 2024-03-07T03:50:28.229360+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2024-03-07T03:50:29.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:29 smithi148 bash[82222]: cluster 2024-03-07T03:50:28.229360+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2024-03-07T03:50:29.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:29 smithi195 bash[66542]: cluster 2024-03-07T03:50:28.229360+0000 mon.a (mon.0) 1939 : cluster 0 osdmap e1082: 8 total, 8 up, 8 in 2024-03-07T03:50:30.298 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:30.298 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:30.326 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:30.327 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:30.327 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:30.335 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:30.335 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:30.335 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:30.343 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:30.343 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:30.343 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:30.351 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:30.351 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:30.351 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:30.358 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:30.358 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:30.358 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:30.365 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:30.365 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:30.365 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:30.372 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:30.372 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:30.372 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:30.378 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:30.378 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:30.378 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:30.384 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:30.384 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:30.384 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:30.389 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:30.389 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:30.389 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:30.392 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:30.392 INFO:tasks.workunit.client.0.smithi148.stdout:from 11 to HEAD diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:30.392 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~12582912,41943040~12582912,58720256~41943040,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~4194304,327155712~20971520,364904448~33554432] 2024-03-07T03:50:30.394 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~58720256,109051904~58720256,171966464~16777216,201326592~4194304,209715200~20971520,234881024~62914560,318767104~29360128,364904448~37748736] 2024-03-07T03:50:30.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:30 smithi148 bash[79819]: cluster 2024-03-07T03:50:29.240863+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2024-03-07T03:50:30.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:30 smithi148 bash[79819]: cluster 2024-03-07T03:50:29.264065+0000 mgr.y (mgr.34101) 1096 : cluster 0 pgmap v1798: 105 pgs: 4 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 99 active+clean; 849 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 391 KiB/s rd, 5.5 KiB/s wr, 707 op/s 2024-03-07T03:50:30.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:30 smithi148 bash[82222]: cluster 2024-03-07T03:50:29.240863+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2024-03-07T03:50:30.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:30 smithi148 bash[82222]: cluster 2024-03-07T03:50:29.264065+0000 mgr.y (mgr.34101) 1096 : cluster 0 pgmap v1798: 105 pgs: 4 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 99 active+clean; 849 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 391 KiB/s rd, 5.5 KiB/s wr, 707 op/s 2024-03-07T03:50:30.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:30 smithi195 bash[66542]: cluster 2024-03-07T03:50:29.240863+0000 mon.a (mon.0) 1940 : cluster 0 osdmap e1083: 8 total, 8 up, 8 in 2024-03-07T03:50:30.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:30 smithi195 bash[66542]: cluster 2024-03-07T03:50:29.264065+0000 mgr.y (mgr.34101) 1096 : cluster 0 pgmap v1798: 105 pgs: 4 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 99 active+clean; 849 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 391 KiB/s rd, 5.5 KiB/s wr, 707 op/s 2024-03-07T03:50:31.310 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:31.310 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:31.335 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:31.335 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:31.335 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:31.342 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:31.342 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:31.342 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:31.349 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:31.349 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:31.349 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:31.355 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:31.356 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:31.356 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:31.362 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:31.362 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:31.362 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:31.367 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:31.367 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:31.367 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:31.371 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:31.372 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:31.372 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:31.375 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:31.376 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:31.376 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:31.379 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:31.379 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:31.379 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:31.382 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:31.382 INFO:tasks.workunit.client.0.smithi148.stdout:from 10 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:31.382 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~12582912,251658240~46137344,318767104~29360128,364904448~33554432] 2024-03-07T03:50:31.384 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,318767104~29360128,364904448~37748736] 2024-03-07T03:50:31.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:31 smithi148 bash[79819]: cluster 2024-03-07T03:50:30.258293+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2024-03-07T03:50:31.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:31 smithi148 bash[82222]: cluster 2024-03-07T03:50:30.258293+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2024-03-07T03:50:31.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:31 smithi195 bash[66542]: cluster 2024-03-07T03:50:30.258293+0000 mon.a (mon.0) 1941 : cluster 0 osdmap e1084: 8 total, 8 up, 8 in 2024-03-07T03:50:32.300 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:32.300 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:32.351 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:32.351 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.351 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.358 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:32.358 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.358 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.364 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:32.364 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.364 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.370 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.370 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.370 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.376 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.376 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.376 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.381 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.381 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.381 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.385 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.385 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.385 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.389 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.389 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:32.389 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:32.391 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:32.392 INFO:tasks.workunit.client.0.smithi148.stdout:from 9 to HEAD diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:32.392 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~4194304,12582912~16777216,41943040~12582912,58720256~29360128,92274688~12582912,109051904~58720256,171966464~4194304,180355072~4194304,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,364904448~37748736] 2024-03-07T03:50:32.394 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,364904448~41943040] 2024-03-07T03:50:32.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:32 smithi148 bash[79819]: cluster 2024-03-07T03:50:31.261482+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2024-03-07T03:50:32.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:32 smithi148 bash[79819]: cluster 2024-03-07T03:50:31.265024+0000 mgr.y (mgr.34101) 1097 : cluster 0 pgmap v1801: 105 pgs: 4 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 99 active+clean; 834 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 220 KiB/s rd, 1.2 KiB/s wr, 440 op/s 2024-03-07T03:50:32.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:32 smithi148 bash[82222]: cluster 2024-03-07T03:50:31.261482+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2024-03-07T03:50:32.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:32 smithi148 bash[82222]: cluster 2024-03-07T03:50:31.265024+0000 mgr.y (mgr.34101) 1097 : cluster 0 pgmap v1801: 105 pgs: 4 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 99 active+clean; 834 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 220 KiB/s rd, 1.2 KiB/s wr, 440 op/s 2024-03-07T03:50:32.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:50:32 smithi195 bash[59134]: ts=2024-03-07T03:50:32.317Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:32.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:32 smithi195 bash[66542]: cluster 2024-03-07T03:50:31.261482+0000 mon.a (mon.0) 1942 : cluster 0 osdmap e1085: 8 total, 8 up, 8 in 2024-03-07T03:50:32.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:32 smithi195 bash[66542]: cluster 2024-03-07T03:50:31.265024+0000 mgr.y (mgr.34101) 1097 : cluster 0 pgmap v1801: 105 pgs: 4 active+clean+snaptrim_wait, 2 active+clean+snaptrim, 99 active+clean; 834 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 220 KiB/s rd, 1.2 KiB/s wr, 440 op/s 2024-03-07T03:50:32.840 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:32.840 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:32.875 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:32.875 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.875 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.882 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:32.882 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.882 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.887 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:32.887 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.887 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:32.893 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.893 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.893 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.897 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.898 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.898 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:32.902 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.902 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.902 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.906 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.906 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.906 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:32.909 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:32.909 INFO:tasks.workunit.client.0.smithi148.stdout:from 8 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:32.909 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,201326592~4194304,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~20971520,352321536~8388608,364904448~37748736] 2024-03-07T03:50:32.911 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,201326592~4194304,209715200~88080384,306184192~41943040,352321536~54525952] 2024-03-07T03:50:33.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:33 smithi148 bash[79819]: cluster 2024-03-07T03:50:32.274758+0000 mon.a (mon.0) 1943 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2024-03-07T03:50:33.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:33 smithi148 bash[79819]: cluster 2024-03-07T03:50:32.801317+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2024-03-07T03:50:33.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:33 smithi148 bash[82222]: cluster 2024-03-07T03:50:32.274758+0000 mon.a (mon.0) 1943 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2024-03-07T03:50:33.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:33 smithi148 bash[82222]: cluster 2024-03-07T03:50:32.801317+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2024-03-07T03:50:33.571 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:33 smithi195 bash[66542]: cluster 2024-03-07T03:50:32.274758+0000 mon.a (mon.0) 1943 : cluster 0 osdmap e1086: 8 total, 8 up, 8 in 2024-03-07T03:50:33.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:33 smithi195 bash[66542]: cluster 2024-03-07T03:50:32.801317+0000 mon.a (mon.0) 1944 : cluster 0 osdmap e1087: 8 total, 8 up, 8 in 2024-03-07T03:50:33.839 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:33.839 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:33.875 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:33.876 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:33.876 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:33.883 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:33.883 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:33.883 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:33.888 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:33.888 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:33.888 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:33.892 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:33.892 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:33.892 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:33.896 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:33.896 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:33.896 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:33.899 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:33.899 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:33.899 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:33.901 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:33.901 INFO:tasks.workunit.client.0.smithi148.stdout:from 7 to HEAD diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:33.902 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~12582912,58720256~46137344,109051904~58720256,171966464~12582912,197132288~8388608,209715200~8388608,222298112~8388608,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:33.903 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~125829120,171966464~16777216,197132288~100663296,306184192~104857600] 2024-03-07T03:50:34.872 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:34.872 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:34.893 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:34.894 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:34.894 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:34.898 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:34.898 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:34.898 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:34.903 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:34.903 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:34.903 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:34.907 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:34.907 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:34.907 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:34.910 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:34.910 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:34.910 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:34.913 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:34.913 INFO:tasks.workunit.client.0.smithi148.stdout:from 6 to HEAD diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:34.913 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~25165824,41943040~62914560,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~16777216,327155712~83886080] 2024-03-07T03:50:34.915 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~25165824,41943040~150994944,197132288~100663296,306184192~104857600] 2024-03-07T03:50:35.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:34 smithi148 bash[79819]: cluster 2024-03-07T03:50:33.266127+0000 mgr.y (mgr.34101) 1098 : cluster 0 pgmap v1804: 105 pgs: 105 active+clean; 782 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 231 KiB/s rd, 5.0 KiB/s wr, 387 op/s 2024-03-07T03:50:35.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:34 smithi148 bash[79819]: cluster 2024-03-07T03:50:33.811412+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2024-03-07T03:50:35.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:34 smithi148 bash[82222]: cluster 2024-03-07T03:50:33.266127+0000 mgr.y (mgr.34101) 1098 : cluster 0 pgmap v1804: 105 pgs: 105 active+clean; 782 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 231 KiB/s rd, 5.0 KiB/s wr, 387 op/s 2024-03-07T03:50:35.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:34 smithi148 bash[82222]: cluster 2024-03-07T03:50:33.811412+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2024-03-07T03:50:35.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:34 smithi195 bash[66542]: cluster 2024-03-07T03:50:33.266127+0000 mgr.y (mgr.34101) 1098 : cluster 0 pgmap v1804: 105 pgs: 105 active+clean; 782 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 231 KiB/s rd, 5.0 KiB/s wr, 387 op/s 2024-03-07T03:50:35.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:34 smithi195 bash[66542]: cluster 2024-03-07T03:50:33.811412+0000 mon.a (mon.0) 1945 : cluster 0 osdmap e1088: 8 total, 8 up, 8 in 2024-03-07T03:50:35.867 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:35.868 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:35.910 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:35.911 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:35.911 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:35.915 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:35.915 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:35.915 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:35.919 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:35.919 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:35.919 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:35.921 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:35.921 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:35.922 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:35.924 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:35.924 INFO:tasks.workunit.client.0.smithi148.stdout:from 5 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:35.924 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:35.926 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:36.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:35 smithi148 bash[79819]: cluster 2024-03-07T03:50:34.814009+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2024-03-07T03:50:36.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:35 smithi148 bash[82222]: cluster 2024-03-07T03:50:34.814009+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2024-03-07T03:50:36.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:35 smithi195 bash[66542]: cluster 2024-03-07T03:50:34.814009+0000 mon.a (mon.0) 1946 : cluster 0 osdmap e1089: 8 total, 8 up, 8 in 2024-03-07T03:50:36.890 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:36.890 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:36.923 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:36.923 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:36.923 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:36.927 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:36.927 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:36.927 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:36.931 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:36.931 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:36.931 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:36.934 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:36.934 INFO:tasks.workunit.client.0.smithi148.stdout:from 4 to HEAD diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:36.934 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~67108864,109051904~58720256,171966464~16777216,197132288~8388608,209715200~20971520,234881024~62914560,306184192~104857600] 2024-03-07T03:50:36.936 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:37.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:36 smithi148 bash[79819]: cluster 2024-03-07T03:50:35.266840+0000 mgr.y (mgr.34101) 1099 : cluster 0 pgmap v1807: 105 pgs: 105 active+clean; 705 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 242 KiB/s rd, 6.5 KiB/s wr, 407 op/s 2024-03-07T03:50:37.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:36 smithi148 bash[79819]: cluster 2024-03-07T03:50:35.843711+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2024-03-07T03:50:37.176 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:36 smithi148 bash[82222]: cluster 2024-03-07T03:50:35.266840+0000 mgr.y (mgr.34101) 1099 : cluster 0 pgmap v1807: 105 pgs: 105 active+clean; 705 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 242 KiB/s rd, 6.5 KiB/s wr, 407 op/s 2024-03-07T03:50:37.176 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:36 smithi148 bash[82222]: cluster 2024-03-07T03:50:35.843711+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2024-03-07T03:50:37.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:36 smithi195 bash[66542]: cluster 2024-03-07T03:50:35.266840+0000 mgr.y (mgr.34101) 1099 : cluster 0 pgmap v1807: 105 pgs: 105 active+clean; 705 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 242 KiB/s rd, 6.5 KiB/s wr, 407 op/s 2024-03-07T03:50:37.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:36 smithi195 bash[66542]: cluster 2024-03-07T03:50:35.843711+0000 mon.a (mon.0) 1947 : cluster 0 osdmap e1090: 8 total, 8 up, 8 in 2024-03-07T03:50:37.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:50:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:50:37] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.43.0" 2024-03-07T03:50:37.819 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:37.819 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:37.884 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:37.884 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:37.884 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:37.888 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:37.888 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:37.888 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:37.891 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:37.891 INFO:tasks.workunit.client.0.smithi148.stdout:from 3 to HEAD diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:37.891 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [4194304~29360128,37748736~130023424,171966464~16777216,197132288~8388608,209715200~83886080,306184192~104857600] 2024-03-07T03:50:37.892 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [4194304~29360128,37748736~155189248,197132288~100663296,306184192~104857600] 2024-03-07T03:50:38.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:37 smithi148 bash[82222]: cluster 2024-03-07T03:50:36.854653+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2024-03-07T03:50:38.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:37 smithi148 bash[82222]: cluster 2024-03-07T03:50:37.803495+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2024-03-07T03:50:38.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:37 smithi148 bash[79819]: cluster 2024-03-07T03:50:36.854653+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2024-03-07T03:50:38.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:37 smithi148 bash[79819]: cluster 2024-03-07T03:50:37.803495+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2024-03-07T03:50:38.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:37 smithi195 bash[66542]: cluster 2024-03-07T03:50:36.854653+0000 mon.a (mon.0) 1948 : cluster 0 osdmap e1091: 8 total, 8 up, 8 in 2024-03-07T03:50:38.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:37 smithi195 bash[66542]: cluster 2024-03-07T03:50:37.803495+0000 mon.a (mon.0) 1949 : cluster 0 osdmap e1092: 8 total, 8 up, 8 in 2024-03-07T03:50:38.908 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:38.908 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:38.933 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:38.933 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:38.933 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:38.937 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:38.937 INFO:tasks.workunit.client.0.smithi148.stdout:from 2 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:38.937 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:38.940 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:39.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:38 smithi148 bash[79819]: cluster 2024-03-07T03:50:37.267633+0000 mgr.y (mgr.34101) 1100 : cluster 0 pgmap v1810: 105 pgs: 105 active+clean; 683 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 6.5 KiB/s wr, 285 op/s 2024-03-07T03:50:39.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:38 smithi148 bash[82222]: cluster 2024-03-07T03:50:37.267633+0000 mgr.y (mgr.34101) 1100 : cluster 0 pgmap v1810: 105 pgs: 105 active+clean; 683 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 6.5 KiB/s wr, 285 op/s 2024-03-07T03:50:39.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:38 smithi195 bash[66542]: cluster 2024-03-07T03:50:37.267633+0000 mgr.y (mgr.34101) 1100 : cluster 0 pgmap v1810: 105 pgs: 105 active+clean; 683 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 170 KiB/s rd, 6.5 KiB/s wr, 285 op/s 2024-03-07T03:50:39.937 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:39.937 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:39.961 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:39.961 INFO:tasks.workunit.client.0.smithi148.stdout:from 1 to HEAD diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:39.961 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~33554432,37748736~130023424,171966464~33554432,209715200~83886080,306184192~104857600] 2024-03-07T03:50:39.963 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~33554432,37748736~260046848,306184192~109051904] 2024-03-07T03:50:40.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:39 smithi148 bash[79819]: cluster 2024-03-07T03:50:38.874471+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2024-03-07T03:50:40.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:39 smithi148 bash[82222]: cluster 2024-03-07T03:50:38.874471+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2024-03-07T03:50:40.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:39 smithi195 bash[66542]: cluster 2024-03-07T03:50:38.874471+0000 mon.a (mon.0) 1950 : cluster 0 osdmap e1093: 8 total, 8 up, 8 in 2024-03-07T03:50:40.940 INFO:tasks.workunit.client.0.smithi148.stdout:from 0 to HEAD diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:40.940 INFO:tasks.workunit.client.0.smithi148.stdout: limited diff [0~167772160,171966464~121634816,301989888~109051904] 2024-03-07T03:50:40.959 INFO:tasks.workunit.client.0.smithi148.stdout: actual was [0~297795584,301989888~113246208] 2024-03-07T03:50:41.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:40 smithi148 bash[79819]: cluster 2024-03-07T03:50:39.268516+0000 mgr.y (mgr.34101) 1101 : cluster 0 pgmap v1813: 105 pgs: 105 active+clean; 590 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 5.7 KiB/s wr, 209 op/s 2024-03-07T03:50:41.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:40 smithi148 bash[79819]: cluster 2024-03-07T03:50:39.893132+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2024-03-07T03:50:41.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:40 smithi148 bash[82222]: cluster 2024-03-07T03:50:39.268516+0000 mgr.y (mgr.34101) 1101 : cluster 0 pgmap v1813: 105 pgs: 105 active+clean; 590 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 5.7 KiB/s wr, 209 op/s 2024-03-07T03:50:41.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:40 smithi148 bash[82222]: cluster 2024-03-07T03:50:39.893132+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2024-03-07T03:50:41.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:40 smithi195 bash[66542]: cluster 2024-03-07T03:50:39.268516+0000 mgr.y (mgr.34101) 1101 : cluster 0 pgmap v1813: 105 pgs: 105 active+clean; 590 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 132 KiB/s rd, 5.7 KiB/s wr, 209 op/s 2024-03-07T03:50:41.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:40 smithi195 bash[66542]: cluster 2024-03-07T03:50:39.893132+0000 mon.a (mon.0) 1951 : cluster 0 osdmap e1094: 8 total, 8 up, 8 in 2024-03-07T03:50:41.962 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (39956 ms) 2024-03-07T03:50:41.962 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-03-07T03:50:41.962 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:42.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:41 smithi148 bash[79819]: cluster 2024-03-07T03:50:40.905066+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2024-03-07T03:50:42.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:41 smithi148 bash[79819]: cluster 2024-03-07T03:50:41.269228+0000 mgr.y (mgr.34101) 1102 : cluster 0 pgmap v1816: 105 pgs: 105 active+clean; 552 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 767 B/s wr, 56 op/s 2024-03-07T03:50:42.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:41 smithi148 bash[82222]: cluster 2024-03-07T03:50:40.905066+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2024-03-07T03:50:42.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:41 smithi148 bash[82222]: cluster 2024-03-07T03:50:41.269228+0000 mgr.y (mgr.34101) 1102 : cluster 0 pgmap v1816: 105 pgs: 105 active+clean; 552 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 767 B/s wr, 56 op/s 2024-03-07T03:50:42.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:41 smithi195 bash[66542]: cluster 2024-03-07T03:50:40.905066+0000 mon.a (mon.0) 1952 : cluster 0 osdmap e1095: 8 total, 8 up, 8 in 2024-03-07T03:50:42.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:41 smithi195 bash[66542]: cluster 2024-03-07T03:50:41.269228+0000 mgr.y (mgr.34101) 1102 : cluster 0 pgmap v1816: 105 pgs: 105 active+clean; 552 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 767 B/s wr, 56 op/s 2024-03-07T03:50:42.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:50:42 smithi195 bash[59134]: ts=2024-03-07T03:50:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:43.064 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1103 ms) 2024-03-07T03:50:43.065 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-03-07T03:50:43.065 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:43.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:42 smithi148 bash[79819]: cluster 2024-03-07T03:50:41.913593+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2024-03-07T03:50:43.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:42 smithi148 bash[79819]: audit 2024-03-07T03:50:42.353884+0000 mon.a (mon.0) 1954 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:43.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:42 smithi148 bash[79819]: cluster 2024-03-07T03:50:42.805987+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2024-03-07T03:50:43.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:42 smithi148 bash[82222]: cluster 2024-03-07T03:50:41.913593+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2024-03-07T03:50:43.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:42 smithi148 bash[82222]: audit 2024-03-07T03:50:42.353884+0000 mon.a (mon.0) 1954 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:43.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:42 smithi148 bash[82222]: cluster 2024-03-07T03:50:42.805987+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2024-03-07T03:50:43.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:42 smithi195 bash[66542]: cluster 2024-03-07T03:50:41.913593+0000 mon.a (mon.0) 1953 : cluster 0 osdmap e1096: 8 total, 8 up, 8 in 2024-03-07T03:50:43.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:42 smithi195 bash[66542]: audit 2024-03-07T03:50:42.353884+0000 mon.a (mon.0) 1954 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:43.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:42 smithi195 bash[66542]: cluster 2024-03-07T03:50:42.805987+0000 mon.a (mon.0) 1955 : cluster 0 osdmap e1097: 8 total, 8 up, 8 in 2024-03-07T03:50:44.199 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [0~20971520] to clone 2024-03-07T03:50:44.200 INFO:tasks.workunit.client.0.smithi148.stdout: diff was [0~20971520] 2024-03-07T03:50:44.209 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1144 ms) 2024-03-07T03:50:44.209 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-03-07T03:50:44.209 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:44.278 INFO:tasks.workunit.client.0.smithi148.stdout: wrote [6070286~99584,8570798~60685,8650517~59860,8828669~64952,15334391~72204,15430417~61878,19344315~91237,20687523~17295] 2024-03-07T03:50:44.291 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (82 ms) 2024-03-07T03:50:44.291 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-03-07T03:50:44.291 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:44.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:43 smithi148 bash[79819]: cluster 2024-03-07T03:50:43.270192+0000 mgr.y (mgr.34101) 1103 : cluster 0 pgmap v1819: 105 pgs: 105 active+clean; 488 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.5 MiB/s wr, 117 op/s 2024-03-07T03:50:44.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:43 smithi148 bash[82222]: cluster 2024-03-07T03:50:43.270192+0000 mgr.y (mgr.34101) 1103 : cluster 0 pgmap v1819: 105 pgs: 105 active+clean; 488 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.5 MiB/s wr, 117 op/s 2024-03-07T03:50:44.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:43 smithi195 bash[66542]: cluster 2024-03-07T03:50:43.270192+0000 mgr.y (mgr.34101) 1103 : cluster 0 pgmap v1819: 105 pgs: 105 active+clean; 488 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.1 MiB/s rd, 2.5 MiB/s wr, 117 op/s 2024-03-07T03:50:45.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:44 smithi148 bash[79819]: cluster 2024-03-07T03:50:43.925059+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2024-03-07T03:50:45.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:44 smithi148 bash[82222]: cluster 2024-03-07T03:50:43.925059+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2024-03-07T03:50:45.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:44 smithi195 bash[66542]: cluster 2024-03-07T03:50:43.925059+0000 mon.a (mon.0) 1956 : cluster 0 osdmap e1098: 8 total, 8 up, 8 in 2024-03-07T03:50:46.175 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1885 ms) 2024-03-07T03:50:46.176 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-03-07T03:50:46.176 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:46.290 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (115 ms) 2024-03-07T03:50:46.291 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-03-07T03:50:46.291 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:46.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:45 smithi148 bash[79819]: cluster 2024-03-07T03:50:44.925823+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2024-03-07T03:50:46.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:45 smithi148 bash[79819]: cluster 2024-03-07T03:50:45.270966+0000 mgr.y (mgr.34101) 1104 : cluster 0 pgmap v1822: 105 pgs: 105 active+clean; 492 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 8.6 MiB/s wr, 184 op/s 2024-03-07T03:50:46.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:45 smithi148 bash[82222]: cluster 2024-03-07T03:50:44.925823+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2024-03-07T03:50:46.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:45 smithi148 bash[82222]: cluster 2024-03-07T03:50:45.270966+0000 mgr.y (mgr.34101) 1104 : cluster 0 pgmap v1822: 105 pgs: 105 active+clean; 492 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 8.6 MiB/s wr, 184 op/s 2024-03-07T03:50:46.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:45 smithi195 bash[66542]: cluster 2024-03-07T03:50:44.925823+0000 mon.a (mon.0) 1957 : cluster 0 osdmap e1099: 8 total, 8 up, 8 in 2024-03-07T03:50:46.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:45 smithi195 bash[66542]: cluster 2024-03-07T03:50:45.270966+0000 mgr.y (mgr.34101) 1104 : cluster 0 pgmap v1822: 105 pgs: 105 active+clean; 492 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 8.6 MiB/s wr, 184 op/s 2024-03-07T03:50:46.412 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (122 ms) 2024-03-07T03:50:46.413 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTryAcquireLock 2024-03-07T03:50:46.413 INFO:tasks.workunit.client.0.smithi148.stdout:using new format! 2024-03-07T03:50:47.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:46 smithi148 bash[79819]: cluster 2024-03-07T03:50:45.947128+0000 mon.a (mon.0) 1958 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2024-03-07T03:50:47.301 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:50:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:50:47] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-03-07T03:50:47.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:46 smithi148 bash[82222]: cluster 2024-03-07T03:50:45.947128+0000 mon.a (mon.0) 1958 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2024-03-07T03:50:47.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:46 smithi195 bash[66542]: cluster 2024-03-07T03:50:45.947128+0000 mon.a (mon.0) 1958 : cluster 0 osdmap e1100: 8 total, 8 up, 8 in 2024-03-07T03:50:49.050 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:48 smithi148 bash[79819]: cluster 2024-03-07T03:50:47.271775+0000 mgr.y (mgr.34101) 1105 : cluster 0 pgmap v1824: 105 pgs: 105 active+clean; 513 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 10 MiB/s wr, 217 op/s 2024-03-07T03:50:49.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:48 smithi148 bash[79819]: cluster 2024-03-07T03:50:47.808080+0000 mon.a (mon.0) 1959 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2024-03-07T03:50:49.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:48 smithi148 bash[82222]: cluster 2024-03-07T03:50:47.271775+0000 mgr.y (mgr.34101) 1105 : cluster 0 pgmap v1824: 105 pgs: 105 active+clean; 513 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 10 MiB/s wr, 217 op/s 2024-03-07T03:50:49.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:48 smithi148 bash[82222]: cluster 2024-03-07T03:50:47.808080+0000 mon.a (mon.0) 1959 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2024-03-07T03:50:49.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:48 smithi195 bash[66542]: cluster 2024-03-07T03:50:47.271775+0000 mgr.y (mgr.34101) 1105 : cluster 0 pgmap v1824: 105 pgs: 105 active+clean; 513 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 10 MiB/s wr, 217 op/s 2024-03-07T03:50:49.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:48 smithi195 bash[66542]: cluster 2024-03-07T03:50:47.808080+0000 mon.a (mon.0) 1959 : cluster 0 osdmap e1101: 8 total, 8 up, 8 in 2024-03-07T03:50:51.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:50 smithi195 bash[66542]: cluster 2024-03-07T03:50:49.272971+0000 mgr.y (mgr.34101) 1106 : cluster 0 pgmap v1826: 105 pgs: 105 active+clean; 572 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 9.0 MiB/s wr, 275 op/s 2024-03-07T03:50:51.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:50 smithi148 bash[79819]: cluster 2024-03-07T03:50:49.272971+0000 mgr.y (mgr.34101) 1106 : cluster 0 pgmap v1826: 105 pgs: 105 active+clean; 572 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 9.0 MiB/s wr, 275 op/s 2024-03-07T03:50:51.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:50 smithi148 bash[82222]: cluster 2024-03-07T03:50:49.272971+0000 mgr.y (mgr.34101) 1106 : cluster 0 pgmap v1826: 105 pgs: 105 active+clean; 572 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.0 MiB/s rd, 9.0 MiB/s wr, 275 op/s 2024-03-07T03:50:51.521 INFO:tasks.workunit.client.0.smithi148.stdout:[ OK ] DiffIterateTest/1.DiffIterateTryAcquireLock (5108 ms) 2024-03-07T03:50:51.521 INFO:tasks.workunit.client.0.smithi148.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-03-07T03:50:51.522 INFO:tasks.workunit.client.0.smithi148.stdout:./src/test/librbd/test_librbd.cc:8289: Skipped 2024-03-07T03:50:51.522 INFO:tasks.workunit.client.0.smithi148.stdout:Skipping due to unmet REQUIRE 2024-03-07T03:50:51.522 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] DiffIterateTest/1.DiffIterateStriping (1 ms) 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] 13 tests from DiffIterateTest/1 (63681 ms total) 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[----------] Global test environment tear-down 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[==========] 476 tests from 24 test suites ran. (2257396 ms total) 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ PASSED ] 430 tests. 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] 46 tests, listed below: 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.AioDiscardEvent 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.AioWriteEvent 2024-03-07T03:50:51.830 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.AioFlushEvent 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapCreate 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapProtect 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapUnprotect 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapRename 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapRollback 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.SnapRemove 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.Rename 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.Resize 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.Flatten 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.UpdateFeatures 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.MetadataSet 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.MetadataRemove 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalReplay.ObjectPosition 2024-03-07T03:50:51.831 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioWrite 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioDiscard 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioDiscardWithPrune 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestJournalEntries.AioFlush 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestOperations.DisableJournalingCorrupt 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringSamePool 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringAbort 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolDisabled 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolEnabled 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMigration.MirroringPool 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-03-07T03:50:51.832 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMirroring.DisableImageMirrorWithPeer 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestMirroring.DisableJournalingWithPeer 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestInternal.DiscardCopyup 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestInternal.TestCoR 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestLibRBD.EncryptionLoadBadStripePattern 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestLibRBD.EnableJournalingViaLockOwner 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] TestLibRBD.ExclusiveLockReadTransition 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] DiffIterateTest/0.DiffIterateStriping 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout:[ SKIPPED ] DiffIterateTest/1.DiffIterateStriping 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:50:51.833 INFO:tasks.workunit.client.0.smithi148.stdout: YOU HAVE 2 DISABLED TESTS 2024-03-07T03:50:51.834 INFO:tasks.workunit.client.0.smithi148.stdout: 2024-03-07T03:50:51.882 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-03-07T03:50:51.882 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-03-07T03:50:51.938 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-03-07T03:50:51.938 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-03-07T03:50:52.217 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:52.217 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-03-07T03:50:52.230 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-03-07T03:50:52.230 DEBUG:teuthology.orchestra.run.smithi148:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-03-07T03:50:52.287 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-03-07T03:50:52.287 INFO:teuthology.task.sequential:In sequential, running task print... 2024-03-07T03:50:52.288 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-03-07T03:50:52.288 DEBUG:teuthology.parallel:result is None 2024-03-07T03:50:52.288 INFO:teuthology.run_tasks:Running task print... 2024-03-07T03:50:52.298 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-03-07T03:50:52.298 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-07T03:50:52.307 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-07T03:50:52.307 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-07T03:50:52.308 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph mgr dump --format=json 2024-03-07T03:50:52.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:50:52 smithi195 bash[59134]: ts=2024-03-07T03:50:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:50:53.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:52 smithi195 bash[66542]: cluster 2024-03-07T03:50:51.273853+0000 mgr.y (mgr.34101) 1107 : cluster 0 pgmap v1827: 105 pgs: 105 active+clean; 578 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.9 MiB/s wr, 226 op/s 2024-03-07T03:50:53.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:52 smithi195 bash[66542]: cluster 2024-03-07T03:50:51.821944+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e1102: 8 total, 8 up, 8 in 2024-03-07T03:50:53.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:52 smithi148 bash[79819]: cluster 2024-03-07T03:50:51.273853+0000 mgr.y (mgr.34101) 1107 : cluster 0 pgmap v1827: 105 pgs: 105 active+clean; 578 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.9 MiB/s wr, 226 op/s 2024-03-07T03:50:53.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:52 smithi148 bash[79819]: cluster 2024-03-07T03:50:51.821944+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e1102: 8 total, 8 up, 8 in 2024-03-07T03:50:53.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:52 smithi148 bash[82222]: cluster 2024-03-07T03:50:51.273853+0000 mgr.y (mgr.34101) 1107 : cluster 0 pgmap v1827: 105 pgs: 105 active+clean; 578 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3.4 MiB/s rd, 6.9 MiB/s wr, 226 op/s 2024-03-07T03:50:53.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:52 smithi148 bash[82222]: cluster 2024-03-07T03:50:51.821944+0000 mon.a (mon.0) 1960 : cluster 0 osdmap e1102: 8 total, 8 up, 8 in 2024-03-07T03:50:55.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:54 smithi195 bash[66542]: cluster 2024-03-07T03:50:53.274685+0000 mgr.y (mgr.34101) 1108 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:50:55.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:54 smithi148 bash[79819]: cluster 2024-03-07T03:50:53.274685+0000 mgr.y (mgr.34101) 1108 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:50:55.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:54 smithi148 bash[82222]: cluster 2024-03-07T03:50:53.274685+0000 mgr.y (mgr.34101) 1108 : cluster 0 pgmap v1829: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 713 GiB / 715 GiB avail 2024-03-07T03:50:57.061 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:50:57.084 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:56 smithi148 bash[79819]: cluster 2024-03-07T03:50:55.275535+0000 mgr.y (mgr.34101) 1109 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:50:57.084 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:56 smithi148 bash[82222]: cluster 2024-03-07T03:50:55.275535+0000 mgr.y (mgr.34101) 1109 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:50:57.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:56 smithi195 bash[66542]: cluster 2024-03-07T03:50:55.275535+0000 mgr.y (mgr.34101) 1109 : cluster 0 pgmap v1830: 73 pgs: 73 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-03-07T03:50:57.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:50:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:50:57] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.43.0" 2024-03-07T03:50:58.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:57 smithi148 bash[79819]: audit 2024-03-07T03:50:57.354179+0000 mon.a (mon.0) 1961 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:58.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:57 smithi148 bash[82222]: audit 2024-03-07T03:50:57.354179+0000 mon.a (mon.0) 1961 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:58.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:57 smithi195 bash[66542]: audit 2024-03-07T03:50:57.354179+0000 mon.a (mon.0) 1961 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:50:59.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:50:58 smithi148 bash[82222]: cluster 2024-03-07T03:50:57.276473+0000 mgr.y (mgr.34101) 1110 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 582 KiB data, 802 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:50:59.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:50:58 smithi148 bash[79819]: cluster 2024-03-07T03:50:57.276473+0000 mgr.y (mgr.34101) 1110 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 582 KiB data, 802 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:50:59.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:50:58 smithi195 bash[66542]: cluster 2024-03-07T03:50:57.276473+0000 mgr.y (mgr.34101) 1110 : cluster 0 pgmap v1831: 73 pgs: 73 active+clean; 582 KiB data, 802 MiB used, 714 GiB / 715 GiB avail 2024-03-07T03:51:00.609 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:51:01.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:00 smithi148 bash[79819]: cluster 2024-03-07T03:50:59.277605+0000 mgr.y (mgr.34101) 1111 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:01.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:00 smithi148 bash[79819]: audit 2024-03-07T03:51:00.605490+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.148:0/822027127' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:51:01.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:00 smithi148 bash[82222]: cluster 2024-03-07T03:50:59.277605+0000 mgr.y (mgr.34101) 1111 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:01.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:00 smithi148 bash[82222]: audit 2024-03-07T03:51:00.605490+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.148:0/822027127' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:51:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:00 smithi195 bash[66542]: cluster 2024-03-07T03:50:59.277605+0000 mgr.y (mgr.34101) 1111 : cluster 0 pgmap v1832: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:00 smithi195 bash[66542]: audit 2024-03-07T03:51:00.605490+0000 mon.c (mon.2) 83 : audit 0 from='client.? 172.21.15.148:0/822027127' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-07T03:51:01.435 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":38,"flags":0,"active_gid":34101,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6800","nonce":3957906672},{"type":"v1","addr":"172.21.15.148:6801","nonce":3957906672}]},"active_addr":"172.21.15.148:6801/3957906672","active_change":"2024-03-07T03:17:12.163132+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":24832,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.148:8443/","prometheus":"http://172.21.15.148:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":286,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":583005969}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":894802874}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":1902083738}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.148:0","nonce":882144892}]}]} 2024-03-07T03:51:01.441 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-07T03:51:01.441 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-07T03:51:01.441 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:51:02.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:51:02 smithi195 bash[59134]: ts=2024-03-07T03:51:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:03.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:02 smithi148 bash[79819]: cluster 2024-03-07T03:51:01.278435+0000 mgr.y (mgr.34101) 1112 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:03.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:02 smithi148 bash[82222]: cluster 2024-03-07T03:51:01.278435+0000 mgr.y (mgr.34101) 1112 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:03.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:02 smithi195 bash[66542]: cluster 2024-03-07T03:51:01.278435+0000 mgr.y (mgr.34101) 1112 : cluster 0 pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:05.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:04 smithi148 bash[79819]: cluster 2024-03-07T03:51:03.279307+0000 mgr.y (mgr.34101) 1113 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:05.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:04 smithi148 bash[82222]: cluster 2024-03-07T03:51:03.279307+0000 mgr.y (mgr.34101) 1113 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:05.321 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:04 smithi195 bash[66542]: cluster 2024-03-07T03:51:03.279307+0000 mgr.y (mgr.34101) 1113 : cluster 0 pgmap v1834: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:06.173 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:07.121 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:06 smithi148 bash[79819]: cluster 2024-03-07T03:51:05.280298+0000 mgr.y (mgr.34101) 1114 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:07.121 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:06 smithi148 bash[82222]: cluster 2024-03-07T03:51:05.280298+0000 mgr.y (mgr.34101) 1114 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:07.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:06 smithi195 bash[66542]: cluster 2024-03-07T03:51:05.280298+0000 mgr.y (mgr.34101) 1114 : cluster 0 pgmap v1835: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:07.427 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:51:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:51:07] "GET /metrics HTTP/1.1" 200 34307 "" "Prometheus/2.43.0" 2024-03-07T03:51:09.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:08 smithi148 bash[79819]: cluster 2024-03-07T03:51:07.280804+0000 mgr.y (mgr.34101) 1115 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:09.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:08 smithi148 bash[82222]: cluster 2024-03-07T03:51:07.280804+0000 mgr.y (mgr.34101) 1115 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:09.160 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:51:09.160 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1102,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:50:51.807402+0000","last_up_change":"2024-03-07T03:25:04.654355+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":49,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:09:29.853395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:11:16.277829+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:11:17.121168+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":463,"up_from":478,"up_thru":1044,"down_at":464,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6803","nonce":3706550114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6805","nonce":3706550114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6809","nonce":3706550114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6807","nonce":3706550114}]},"public_addr":"172.21.15.148:6803/3706550114","cluster_addr":"172.21.15.148:6805/3706550114","heartbeat_back_addr":"172.21.15.148:6809/3706550114","heartbeat_front_addr":"172.21.15.148:6807/3706550114","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":495,"up_from":509,"up_thru":1044,"down_at":496,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6811","nonce":2696710856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6813","nonce":2696710856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6817","nonce":2696710856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6815","nonce":2696710856}]},"public_addr":"172.21.15.148:6811/2696710856","cluster_addr":"172.21.15.148:6813/2696710856","heartbeat_back_addr":"172.21.15.148:6817/2696710856","heartbeat_front_addr":"172.21.15.148:6815/2696710856","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":627,"up_from":634,"up_thru":1044,"down_at":627,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6819","nonce":2867443140}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6821","nonce":2867443140}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6825","nonce":2867443140}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6823","nonce":2867443140}]},"public_addr":"172.21.15.148:6819/2867443140","cluster_addr":"172.21.15.148:6821/2867443140","heartbeat_back_addr":"172.21.15.148:6825/2867443140","heartbeat_front_addr":"172.21.15.148:6823/2867443140","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":646,"up_from":651,"up_thru":1044,"down_at":647,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6827","nonce":3829359435}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6829","nonce":3829359435}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6833","nonce":3829359435}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6831","nonce":3829359435}]},"public_addr":"172.21.15.148:6827/3829359435","cluster_addr":"172.21.15.148:6829/3829359435","heartbeat_back_addr":"172.21.15.148:6833/3829359435","heartbeat_front_addr":"172.21.15.148:6831/3829359435","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":652,"up_from":656,"up_thru":1044,"down_at":653,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6801","nonce":4067214799}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6803","nonce":4067214799}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6807","nonce":4067214799}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6805","nonce":4067214799}]},"public_addr":"172.21.15.195:6801/4067214799","cluster_addr":"172.21.15.195:6803/4067214799","heartbeat_back_addr":"172.21.15.195:6807/4067214799","heartbeat_front_addr":"172.21.15.195:6805/4067214799","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1044,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1044,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1044,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":463},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":494},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":626},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":646},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":652},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:0/2471490164":"2024-03-07T04:31:16.792541+0000","172.21.15.148:6801/1080862179":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/4143961239":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.195:0/241830247":"2024-03-08T03:16:06.657875+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2057842025":"2024-03-08T03:15:11.007593+0000","172.21.15.195:6834/1471704062":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/3198375452":"2024-03-07T04:22:12.164321+0000","172.21.15.148:0/2300269287":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/3482682182":"2024-03-08T03:15:11.007593+0000","172.21.15.148:6834/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:6800/1790474515":"2024-03-08T03:15:11.007593+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000","172.21.15.195:0/855882029":"2024-03-08T03:16:06.657875+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2258249874":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/1790474515":"2024-03-08T03:15:11.007593+0000","172.21.15.148:0/3220133019":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2001636836":"2024-03-08T03:15:11.007593+0000","172.21.15.148:6835/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:0/592735695":"2024-03-08T03:17:12.162464+0000","172.21.15.195:6835/1471704062":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/1314748318":"2024-03-08T03:15:11.007593+0000","172.21.15.195:0/2876849":"2024-03-08T03:16:06.657875+0000","172.21.15.195:0/227219641":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/1901736382":"2024-03-08T03:08:48.946763+0000","172.21.15.195:0/2159933630":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/2064188301":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/2190593811":"2024-03-07T04:22:11.249824+0000","172.21.15.195:0/3278313413":"2024-03-08T03:16:06.657875+0000","172.21.15.148:6800/1080862179":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6800/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3019773952":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/1915485540":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/1382963822":"2024-03-08T03:17:12.162464+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":5,"length":4}]},{"pool":16,"snaps":[{"begin":34,"length":4},{"begin":42,"length":1},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":1},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":49,"snaps":[{"begin":14,"length":2}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:51:09.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:08 smithi195 bash[66542]: cluster 2024-03-07T03:51:07.280804+0000 mgr.y (mgr.34101) 1115 : cluster 0 pgmap v1836: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:09.916 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-07T03:51:09.917 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd dump --format=json 2024-03-07T03:51:10.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:09 smithi148 bash[79819]: audit 2024-03-07T03:51:09.160543+0000 mon.a (mon.0) 1962 : audit 0 from='client.? 172.21.15.148:0/3592703156' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:51:10.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:09 smithi148 bash[82222]: audit 2024-03-07T03:51:09.160543+0000 mon.a (mon.0) 1962 : audit 0 from='client.? 172.21.15.148:0/3592703156' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:51:10.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:09 smithi195 bash[66542]: audit 2024-03-07T03:51:09.160543+0000 mon.a (mon.0) 1962 : audit 0 from='client.? 172.21.15.148:0/3592703156' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:51:11.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:10 smithi148 bash[79819]: cluster 2024-03-07T03:51:09.281281+0000 mgr.y (mgr.34101) 1116 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:11.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:10 smithi148 bash[82222]: cluster 2024-03-07T03:51:09.281281+0000 mgr.y (mgr.34101) 1116 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:11.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:10 smithi195 bash[66542]: cluster 2024-03-07T03:51:09.281281+0000 mgr.y (mgr.34101) 1116 : cluster 0 pgmap v1837: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:12.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:51:12 smithi195 bash[59134]: ts=2024-03-07T03:51:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:13.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:12 smithi148 bash[79819]: cluster 2024-03-07T03:51:11.282023+0000 mgr.y (mgr.34101) 1117 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:13.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:12 smithi148 bash[79819]: audit 2024-03-07T03:51:12.354636+0000 mon.a (mon.0) 1963 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:13.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:12 smithi148 bash[82222]: cluster 2024-03-07T03:51:11.282023+0000 mgr.y (mgr.34101) 1117 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:13.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:12 smithi148 bash[82222]: audit 2024-03-07T03:51:12.354636+0000 mon.a (mon.0) 1963 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:13.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:12 smithi195 bash[66542]: cluster 2024-03-07T03:51:11.282023+0000 mgr.y (mgr.34101) 1117 : cluster 0 pgmap v1838: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:13.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:12 smithi195 bash[66542]: audit 2024-03-07T03:51:12.354636+0000 mon.a (mon.0) 1963 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:14.650 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:15.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:14 smithi148 bash[82222]: cluster 2024-03-07T03:51:13.282913+0000 mgr.y (mgr.34101) 1118 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:15.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:14 smithi148 bash[79819]: cluster 2024-03-07T03:51:13.282913+0000 mgr.y (mgr.34101) 1118 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:15.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:14 smithi195 bash[66542]: cluster 2024-03-07T03:51:13.282913+0000 mgr.y (mgr.34101) 1118 : cluster 0 pgmap v1839: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:17.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:16 smithi148 bash[79819]: cluster 2024-03-07T03:51:15.283829+0000 mgr.y (mgr.34101) 1119 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:17.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:16 smithi148 bash[82222]: cluster 2024-03-07T03:51:15.283829+0000 mgr.y (mgr.34101) 1119 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:17.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:16 smithi195 bash[66542]: cluster 2024-03-07T03:51:15.283829+0000 mgr.y (mgr.34101) 1119 : cluster 0 pgmap v1840: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:17.550 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:51:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:51:17] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:51:17.694 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:51:17.695 INFO:teuthology.orchestra.run.smithi148.stdout:{"epoch":1102,"fsid":"b9c09ac8-dc2d-11ee-95c6-87774f69a715","created":"2024-03-07T02:58:42.789402+0000","modified":"2024-03-07T03:50:51.807402+0000","last_up_change":"2024-03-07T03:25:04.654355+0000","last_in_change":"2024-03-07T03:07:27.088743+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":49,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-07T03:03:43.780289+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-07T03:09:29.853395+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-07T03:11:16.277829+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-07T03:11:17.121168+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6f1dd05f-29db-4a04-8e52-e0c19aeaf719","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":463,"up_from":478,"up_thru":1044,"down_at":464,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6802","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6803","nonce":3706550114}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6804","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6805","nonce":3706550114}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6808","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6809","nonce":3706550114}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6806","nonce":3706550114},{"type":"v1","addr":"172.21.15.148:6807","nonce":3706550114}]},"public_addr":"172.21.15.148:6803/3706550114","cluster_addr":"172.21.15.148:6805/3706550114","heartbeat_back_addr":"172.21.15.148:6809/3706550114","heartbeat_front_addr":"172.21.15.148:6807/3706550114","state":["exists","up"]},{"osd":1,"uuid":"8a945bfb-2341-49b2-94cf-6004ef76162f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":495,"up_from":509,"up_thru":1044,"down_at":496,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6810","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6811","nonce":2696710856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6812","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6813","nonce":2696710856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6816","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6817","nonce":2696710856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6814","nonce":2696710856},{"type":"v1","addr":"172.21.15.148:6815","nonce":2696710856}]},"public_addr":"172.21.15.148:6811/2696710856","cluster_addr":"172.21.15.148:6813/2696710856","heartbeat_back_addr":"172.21.15.148:6817/2696710856","heartbeat_front_addr":"172.21.15.148:6815/2696710856","state":["exists","up"]},{"osd":2,"uuid":"6bb58efa-e4f1-415a-95d5-f4b0df042e11","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":627,"up_from":634,"up_thru":1044,"down_at":627,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6818","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6819","nonce":2867443140}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6820","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6821","nonce":2867443140}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6824","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6825","nonce":2867443140}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6822","nonce":2867443140},{"type":"v1","addr":"172.21.15.148:6823","nonce":2867443140}]},"public_addr":"172.21.15.148:6819/2867443140","cluster_addr":"172.21.15.148:6821/2867443140","heartbeat_back_addr":"172.21.15.148:6825/2867443140","heartbeat_front_addr":"172.21.15.148:6823/2867443140","state":["exists","up"]},{"osd":3,"uuid":"ed9035a7-5b7b-400e-9293-349d2b59c6a3","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":646,"up_from":651,"up_thru":1044,"down_at":647,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6826","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6827","nonce":3829359435}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6828","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6829","nonce":3829359435}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6832","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6833","nonce":3829359435}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.148:6830","nonce":3829359435},{"type":"v1","addr":"172.21.15.148:6831","nonce":3829359435}]},"public_addr":"172.21.15.148:6827/3829359435","cluster_addr":"172.21.15.148:6829/3829359435","heartbeat_back_addr":"172.21.15.148:6833/3829359435","heartbeat_front_addr":"172.21.15.148:6831/3829359435","state":["exists","up"]},{"osd":4,"uuid":"e147b71e-ab98-4dca-ab48-49b4af4fc002","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":652,"up_from":656,"up_thru":1044,"down_at":653,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6800","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6801","nonce":4067214799}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6802","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6803","nonce":4067214799}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6806","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6807","nonce":4067214799}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6804","nonce":4067214799},{"type":"v1","addr":"172.21.15.195:6805","nonce":4067214799}]},"public_addr":"172.21.15.195:6801/4067214799","cluster_addr":"172.21.15.195:6803/4067214799","heartbeat_back_addr":"172.21.15.195:6807/4067214799","heartbeat_front_addr":"172.21.15.195:6805/4067214799","state":["exists","up"]},{"osd":5,"uuid":"c3b9d5d7-331d-4007-9edf-820635cf80f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1044,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6808","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6809","nonce":3610217436}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6810","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6811","nonce":3610217436}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6814","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6815","nonce":3610217436}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6812","nonce":3610217436},{"type":"v1","addr":"172.21.15.195:6813","nonce":3610217436}]},"public_addr":"172.21.15.195:6809/3610217436","cluster_addr":"172.21.15.195:6811/3610217436","heartbeat_back_addr":"172.21.15.195:6815/3610217436","heartbeat_front_addr":"172.21.15.195:6813/3610217436","state":["exists","up"]},{"osd":6,"uuid":"3676cb41-e06f-41d8-b9cc-0271403b23a8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1044,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6816","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6817","nonce":4169107669}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6818","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6819","nonce":4169107669}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6822","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6823","nonce":4169107669}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6820","nonce":4169107669},{"type":"v1","addr":"172.21.15.195:6821","nonce":4169107669}]},"public_addr":"172.21.15.195:6817/4169107669","cluster_addr":"172.21.15.195:6819/4169107669","heartbeat_back_addr":"172.21.15.195:6823/4169107669","heartbeat_front_addr":"172.21.15.195:6821/4169107669","state":["exists","up"]},{"osd":7,"uuid":"6c01c7ac-d80e-48bb-a060-7dc2e2d22e41","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1044,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6824","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6825","nonce":2298968566}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6826","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6827","nonce":2298968566}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6830","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6831","nonce":2298968566}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.195:6828","nonce":2298968566},{"type":"v1","addr":"172.21.15.195:6829","nonce":2298968566}]},"public_addr":"172.21.15.195:6825/2298968566","cluster_addr":"172.21.15.195:6827/2298968566","heartbeat_back_addr":"172.21.15.195:6831/2298968566","heartbeat_front_addr":"172.21.15.195:6829/2298968566","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:03.962367+0000","dead_epoch":463},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:02:51.631835+0000","dead_epoch":494},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:03:40.923289+0000","dead_epoch":626},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:04:29.968366+0000","dead_epoch":646},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:05:18.638900+0000","dead_epoch":652},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:08.576131+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:06:59.116126+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-03-07T03:07:51.810771+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.4","mappings":[{"from":1,"to":5}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.148:0/2471490164":"2024-03-07T04:31:16.792541+0000","172.21.15.148:6801/1080862179":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/4143961239":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.195:0/241830247":"2024-03-08T03:16:06.657875+0000","172.21.15.148:6801/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/1873421446":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2057842025":"2024-03-08T03:15:11.007593+0000","172.21.15.195:6834/1471704062":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/3198375452":"2024-03-07T04:22:12.164321+0000","172.21.15.148:0/2300269287":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/3482682182":"2024-03-08T03:15:11.007593+0000","172.21.15.148:6834/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:6800/1790474515":"2024-03-08T03:15:11.007593+0000","172.21.15.148:0/3864405911":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2699698695":"2024-03-08T02:59:43.137803+0000","172.21.15.195:0/855882029":"2024-03-08T03:16:06.657875+0000","172.21.15.148:6801/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/4151172111":"2024-03-08T02:59:13.038358+0000","172.21.15.148:6800/1609459980":"2024-03-08T02:59:43.137803+0000","172.21.15.148:0/2258249874":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6801/1790474515":"2024-03-08T03:15:11.007593+0000","172.21.15.148:0/3220133019":"2024-03-08T03:08:48.946763+0000","172.21.15.148:0/991175857":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/2001636836":"2024-03-08T03:15:11.007593+0000","172.21.15.148:6835/1878290118":"2024-03-08T03:11:25.123773+0000","172.21.15.148:0/592735695":"2024-03-08T03:17:12.162464+0000","172.21.15.195:6835/1471704062":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/1314748318":"2024-03-08T03:15:11.007593+0000","172.21.15.195:0/2876849":"2024-03-08T03:16:06.657875+0000","172.21.15.195:0/227219641":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/1901736382":"2024-03-08T03:08:48.946763+0000","172.21.15.195:0/2159933630":"2024-03-08T03:16:06.657875+0000","172.21.15.148:0/2064188301":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/2190593811":"2024-03-07T04:22:11.249824+0000","172.21.15.195:0/3278313413":"2024-03-08T03:16:06.657875+0000","172.21.15.148:6800/1080862179":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/2929246662":"2024-03-08T02:59:43.137803+0000","172.21.15.148:6800/480238330":"2024-03-08T03:08:48.946763+0000","172.21.15.148:6800/4155780354":"2024-03-08T02:59:13.038358+0000","172.21.15.148:0/3019773952":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/1915485540":"2024-03-08T03:17:12.162464+0000","172.21.15.148:0/1382963822":"2024-03-08T03:17:12.162464+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":14,"snaps":[{"begin":5,"length":4}]},{"pool":16,"snaps":[{"begin":34,"length":4},{"begin":42,"length":1},{"begin":44,"length":4}]},{"pool":17,"snaps":[{"begin":17,"length":1},{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":49,"snaps":[{"begin":14,"length":2}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-07T03:51:18.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:17 smithi148 bash[79819]: audit 2024-03-07T03:51:17.694088+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.148:0/1343811089' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:51:18.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:17 smithi148 bash[82222]: audit 2024-03-07T03:51:17.694088+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.148:0/1343811089' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:51:18.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:17 smithi195 bash[66542]: audit 2024-03-07T03:51:17.694088+0000 mon.a (mon.0) 1964 : audit 0 from='client.? 172.21.15.148:0/1343811089' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-07T03:51:18.434 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-07T03:51:18.435 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-07T03:51:18.435 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-07T03:51:18.436 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-07T03:51:18.436 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-07T03:51:18.437 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-07T03:51:18.437 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-07T03:51:18.437 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-07T03:51:19.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:18 smithi148 bash[82222]: cluster 2024-03-07T03:51:17.284609+0000 mgr.y (mgr.34101) 1120 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:19.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:18 smithi148 bash[82222]: audit 2024-03-07T03:51:18.766752+0000 mon.a (mon.0) 1965 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:51:19.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:18 smithi148 bash[79819]: cluster 2024-03-07T03:51:17.284609+0000 mgr.y (mgr.34101) 1120 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:19.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:18 smithi148 bash[79819]: audit 2024-03-07T03:51:18.766752+0000 mon.a (mon.0) 1965 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:51:19.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:18 smithi195 bash[66542]: cluster 2024-03-07T03:51:17.284609+0000 mgr.y (mgr.34101) 1120 : cluster 0 pgmap v1841: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:19.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:18 smithi195 bash[66542]: audit 2024-03-07T03:51:18.766752+0000 mon.a (mon.0) 1965 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:51:20.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:20 smithi148 bash[79819]: cluster 2024-03-07T03:51:19.285673+0000 mgr.y (mgr.34101) 1121 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:20.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:20 smithi148 bash[79819]: audit 2024-03-07T03:51:19.291908+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:20.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:20 smithi148 bash[79819]: audit 2024-03-07T03:51:19.297976+0000 mon.a (mon.0) 1967 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:20.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:20 smithi148 bash[82222]: cluster 2024-03-07T03:51:19.285673+0000 mgr.y (mgr.34101) 1121 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:20.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:20 smithi148 bash[82222]: audit 2024-03-07T03:51:19.291908+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:20.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:20 smithi148 bash[82222]: audit 2024-03-07T03:51:19.297976+0000 mon.a (mon.0) 1967 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:20.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:20 smithi195 bash[66542]: cluster 2024-03-07T03:51:19.285673+0000 mgr.y (mgr.34101) 1121 : cluster 0 pgmap v1842: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:20.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:20 smithi195 bash[66542]: audit 2024-03-07T03:51:19.291908+0000 mon.a (mon.0) 1966 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:20.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:20 smithi195 bash[66542]: audit 2024-03-07T03:51:19.297976+0000 mon.a (mon.0) 1967 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:22.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:22 smithi148 bash[79819]: cluster 2024-03-07T03:51:21.286308+0000 mgr.y (mgr.34101) 1122 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:22.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:22 smithi148 bash[82222]: cluster 2024-03-07T03:51:21.286308+0000 mgr.y (mgr.34101) 1122 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:22.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:22 smithi195 bash[66542]: cluster 2024-03-07T03:51:21.286308+0000 mgr.y (mgr.34101) 1122 : cluster 0 pgmap v1843: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:22.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:51:22 smithi195 bash[59134]: ts=2024-03-07T03:51:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:23.251 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.252 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.252 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.252 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.254 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.256 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.256 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:23.258 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:24.550 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:24 smithi148 bash[79819]: cluster 2024-03-07T03:51:23.286937+0000 mgr.y (mgr.34101) 1123 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:24.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:24 smithi148 bash[82222]: cluster 2024-03-07T03:51:23.286937+0000 mgr.y (mgr.34101) 1123 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:24.572 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:24 smithi195 bash[66542]: cluster 2024-03-07T03:51:23.286937+0000 mgr.y (mgr.34101) 1123 : cluster 0 pgmap v1844: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:26.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[79819]: audit 2024-03-07T03:51:24.764469+0000 mon.a (mon.0) 1968 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[79819]: audit 2024-03-07T03:51:24.770768+0000 mon.a (mon.0) 1969 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[79819]: audit 2024-03-07T03:51:25.092168+0000 mon.a (mon.0) 1970 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[79819]: audit 2024-03-07T03:51:25.099270+0000 mon.a (mon.0) 1971 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[79819]: audit 2024-03-07T03:51:25.678371+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[79819]: audit 2024-03-07T03:51:25.680130+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[79819]: audit 2024-03-07T03:51:25.690105+0000 mon.a (mon.0) 1974 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[82222]: audit 2024-03-07T03:51:24.764469+0000 mon.a (mon.0) 1968 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[82222]: audit 2024-03-07T03:51:24.770768+0000 mon.a (mon.0) 1969 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[82222]: audit 2024-03-07T03:51:25.092168+0000 mon.a (mon.0) 1970 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[82222]: audit 2024-03-07T03:51:25.099270+0000 mon.a (mon.0) 1971 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[82222]: audit 2024-03-07T03:51:25.678371+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:51:26.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[82222]: audit 2024-03-07T03:51:25.680130+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:51:26.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:25 smithi148 bash[82222]: audit 2024-03-07T03:51:25.690105+0000 mon.a (mon.0) 1974 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:25 smithi195 bash[66542]: audit 2024-03-07T03:51:24.764469+0000 mon.a (mon.0) 1968 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:25 smithi195 bash[66542]: audit 2024-03-07T03:51:24.770768+0000 mon.a (mon.0) 1969 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:25 smithi195 bash[66542]: audit 2024-03-07T03:51:25.092168+0000 mon.a (mon.0) 1970 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:25 smithi195 bash[66542]: audit 2024-03-07T03:51:25.099270+0000 mon.a (mon.0) 1971 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:26.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:25 smithi195 bash[66542]: audit 2024-03-07T03:51:25.678371+0000 mon.a (mon.0) 1972 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:51:26.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:25 smithi195 bash[66542]: audit 2024-03-07T03:51:25.680130+0000 mon.a (mon.0) 1973 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:51:26.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:25 smithi195 bash[66542]: audit 2024-03-07T03:51:25.690105+0000 mon.a (mon.0) 1974 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:51:27.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:26 smithi148 bash[79819]: cluster 2024-03-07T03:51:25.287499+0000 mgr.y (mgr.34101) 1124 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:27.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:26 smithi148 bash[82222]: cluster 2024-03-07T03:51:25.287499+0000 mgr.y (mgr.34101) 1124 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:27.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:26 smithi195 bash[66542]: cluster 2024-03-07T03:51:25.287499+0000 mgr.y (mgr.34101) 1124 : cluster 0 pgmap v1845: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:27.551 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:51:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:51:27] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:51:28.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:27 smithi148 bash[79819]: audit 2024-03-07T03:51:27.355133+0000 mon.a (mon.0) 1975 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:28.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:27 smithi148 bash[82222]: audit 2024-03-07T03:51:27.355133+0000 mon.a (mon.0) 1975 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:28.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:27 smithi195 bash[66542]: audit 2024-03-07T03:51:27.355133+0000 mon.a (mon.0) 1975 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:29.025 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:28 smithi148 bash[79819]: cluster 2024-03-07T03:51:27.288226+0000 mgr.y (mgr.34101) 1125 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:29.025 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:28 smithi148 bash[82222]: cluster 2024-03-07T03:51:27.288226+0000 mgr.y (mgr.34101) 1125 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:29.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:28 smithi195 bash[66542]: cluster 2024-03-07T03:51:27.288226+0000 mgr.y (mgr.34101) 1125 : cluster 0 pgmap v1846: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:31.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:30 smithi148 bash[79819]: cluster 2024-03-07T03:51:29.288969+0000 mgr.y (mgr.34101) 1126 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:31.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:30 smithi148 bash[82222]: cluster 2024-03-07T03:51:29.288969+0000 mgr.y (mgr.34101) 1126 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:31.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:30 smithi195 bash[66542]: cluster 2024-03-07T03:51:29.288969+0000 mgr.y (mgr.34101) 1126 : cluster 0 pgmap v1847: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:32.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:51:32 smithi195 bash[59134]: ts=2024-03-07T03:51:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:33.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:32 smithi148 bash[79819]: cluster 2024-03-07T03:51:31.289517+0000 mgr.y (mgr.34101) 1127 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:33.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:32 smithi148 bash[82222]: cluster 2024-03-07T03:51:31.289517+0000 mgr.y (mgr.34101) 1127 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:33.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:32 smithi195 bash[66542]: cluster 2024-03-07T03:51:31.289517+0000 mgr.y (mgr.34101) 1127 : cluster 0 pgmap v1848: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:35.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:34 smithi148 bash[79819]: cluster 2024-03-07T03:51:33.290035+0000 mgr.y (mgr.34101) 1128 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:35.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:34 smithi148 bash[82222]: cluster 2024-03-07T03:51:33.290035+0000 mgr.y (mgr.34101) 1128 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:35.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:34 smithi195 bash[66542]: cluster 2024-03-07T03:51:33.290035+0000 mgr.y (mgr.34101) 1128 : cluster 0 pgmap v1849: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:36.094 INFO:teuthology.orchestra.run.smithi148.stdout:2817498546495 2024-03-07T03:51:36.095 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-07T03:51:37.044 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:36 smithi148 bash[82222]: cluster 2024-03-07T03:51:35.290954+0000 mgr.y (mgr.34101) 1129 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:37.044 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:36 smithi148 bash[79819]: cluster 2024-03-07T03:51:35.290954+0000 mgr.y (mgr.34101) 1129 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:37.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:36 smithi195 bash[66542]: cluster 2024-03-07T03:51:35.290954+0000 mgr.y (mgr.34101) 1129 : cluster 0 pgmap v1850: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:37.299 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:51:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:51:37] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.43.0" 2024-03-07T03:51:38.977 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:38 smithi148 bash[79819]: cluster 2024-03-07T03:51:37.291432+0000 mgr.y (mgr.34101) 1130 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:38.977 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:38 smithi148 bash[82222]: cluster 2024-03-07T03:51:37.291432+0000 mgr.y (mgr.34101) 1130 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:39.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:38 smithi195 bash[66542]: cluster 2024-03-07T03:51:37.291432+0000 mgr.y (mgr.34101) 1130 : cluster 0 pgmap v1851: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:39.481 INFO:teuthology.orchestra.run.smithi148.stdout:2796023710022 2024-03-07T03:51:39.482 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-07T03:51:39.603 INFO:teuthology.orchestra.run.smithi148.stdout:2186138354026 2024-03-07T03:51:39.604 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-07T03:51:39.605 INFO:teuthology.orchestra.run.smithi148.stdout:167503725092 2024-03-07T03:51:39.605 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-07T03:51:39.777 INFO:teuthology.orchestra.run.smithi148.stdout:236223201807 2024-03-07T03:51:39.777 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-07T03:51:40.689 INFO:teuthology.orchestra.run.smithi148.stdout:2723009265999 2024-03-07T03:51:40.690 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-07T03:51:40.743 INFO:teuthology.orchestra.run.smithi148.stdout:201863463450 2024-03-07T03:51:40.743 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-07T03:51:40.790 INFO:teuthology.orchestra.run.smithi148.stdout:2052994367856 2024-03-07T03:51:40.791 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-07T03:51:41.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:40 smithi148 bash[82222]: cluster 2024-03-07T03:51:39.292073+0000 mgr.y (mgr.34101) 1131 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:41.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:40 smithi148 bash[79819]: cluster 2024-03-07T03:51:39.292073+0000 mgr.y (mgr.34101) 1131 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:41.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:40 smithi195 bash[66542]: cluster 2024-03-07T03:51:39.292073+0000 mgr.y (mgr.34101) 1131 : cluster 0 pgmap v1852: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:42.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:51:42 smithi195 bash[59134]: ts=2024-03-07T03:51:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:43.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:42 smithi148 bash[79819]: cluster 2024-03-07T03:51:41.292732+0000 mgr.y (mgr.34101) 1132 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:43.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:42 smithi148 bash[79819]: audit 2024-03-07T03:51:42.355516+0000 mon.a (mon.0) 1976 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:43.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:42 smithi148 bash[82222]: cluster 2024-03-07T03:51:41.292732+0000 mgr.y (mgr.34101) 1132 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:43.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:42 smithi148 bash[82222]: audit 2024-03-07T03:51:42.355516+0000 mon.a (mon.0) 1976 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:43.071 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:43.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:42 smithi195 bash[66542]: cluster 2024-03-07T03:51:41.292732+0000 mgr.y (mgr.34101) 1132 : cluster 0 pgmap v1853: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:43.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:42 smithi195 bash[66542]: audit 2024-03-07T03:51:42.355516+0000 mon.a (mon.0) 1976 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:44.089 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:44.089 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:44.090 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:44.092 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:45.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:44 smithi195 bash[66542]: cluster 2024-03-07T03:51:43.293561+0000 mgr.y (mgr.34101) 1133 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:45.094 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:44 smithi148 bash[79819]: cluster 2024-03-07T03:51:43.293561+0000 mgr.y (mgr.34101) 1133 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:45.094 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:44 smithi148 bash[82222]: cluster 2024-03-07T03:51:43.293561+0000 mgr.y (mgr.34101) 1133 : cluster 0 pgmap v1854: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:46.290 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:46.290 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:46.291 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:51:47.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:46 smithi195 bash[66542]: cluster 2024-03-07T03:51:45.294476+0000 mgr.y (mgr.34101) 1134 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:47.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:46 smithi148 bash[79819]: cluster 2024-03-07T03:51:45.294476+0000 mgr.y (mgr.34101) 1134 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:47.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:46 smithi148 bash[82222]: cluster 2024-03-07T03:51:45.294476+0000 mgr.y (mgr.34101) 1134 : cluster 0 pgmap v1855: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:47.479 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:51:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:51:47] "GET /metrics HTTP/1.1" 200 34306 "" "Prometheus/2.43.0" 2024-03-07T03:51:49.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:48 smithi195 bash[66542]: cluster 2024-03-07T03:51:47.294970+0000 mgr.y (mgr.34101) 1135 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:49.086 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:48 smithi148 bash[79819]: cluster 2024-03-07T03:51:47.294970+0000 mgr.y (mgr.34101) 1135 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:49.086 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:48 smithi148 bash[82222]: cluster 2024-03-07T03:51:47.294970+0000 mgr.y (mgr.34101) 1135 : cluster 0 pgmap v1856: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:51.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:50 smithi195 bash[66542]: cluster 2024-03-07T03:51:49.295911+0000 mgr.y (mgr.34101) 1136 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:51.095 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:50 smithi148 bash[79819]: cluster 2024-03-07T03:51:49.295911+0000 mgr.y (mgr.34101) 1136 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:51.095 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:50 smithi148 bash[82222]: cluster 2024-03-07T03:51:49.295911+0000 mgr.y (mgr.34101) 1136 : cluster 0 pgmap v1857: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:51.882 INFO:teuthology.orchestra.run.smithi148.stdout:2817498546499 2024-03-07T03:51:52.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:51:52 smithi195 bash[59134]: ts=2024-03-07T03:51:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:51:53.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:52 smithi195 bash[66542]: cluster 2024-03-07T03:51:51.296730+0000 mgr.y (mgr.34101) 1137 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:53.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:52 smithi195 bash[66542]: audit 2024-03-07T03:51:51.884660+0000 mon.a (mon.0) 1977 : audit 0 from='client.? 172.21.15.148:0/3590005851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:51:53.138 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:52 smithi148 bash[79819]: cluster 2024-03-07T03:51:51.296730+0000 mgr.y (mgr.34101) 1137 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:53.138 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:52 smithi148 bash[79819]: audit 2024-03-07T03:51:51.884660+0000 mon.a (mon.0) 1977 : audit 0 from='client.? 172.21.15.148:0/3590005851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:51:53.139 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:52 smithi148 bash[82222]: cluster 2024-03-07T03:51:51.296730+0000 mgr.y (mgr.34101) 1137 : cluster 0 pgmap v1858: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:53.139 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:52 smithi148 bash[82222]: audit 2024-03-07T03:51:51.884660+0000 mon.a (mon.0) 1977 : audit 0 from='client.? 172.21.15.148:0/3590005851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-07T03:51:54.280 INFO:teuthology.orchestra.run.smithi148.stdout:2186138354029 2024-03-07T03:51:55.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:54 smithi148 bash[79819]: cluster 2024-03-07T03:51:53.297317+0000 mgr.y (mgr.34101) 1138 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:55.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:54 smithi148 bash[79819]: audit 2024-03-07T03:51:54.282350+0000 mon.a (mon.0) 1978 : audit 0 from='client.? 172.21.15.148:0/427565064' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:51:55.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:54 smithi148 bash[82222]: cluster 2024-03-07T03:51:53.297317+0000 mgr.y (mgr.34101) 1138 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:55.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:54 smithi148 bash[82222]: audit 2024-03-07T03:51:54.282350+0000 mon.a (mon.0) 1978 : audit 0 from='client.? 172.21.15.148:0/427565064' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:51:55.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:54 smithi195 bash[66542]: cluster 2024-03-07T03:51:53.297317+0000 mgr.y (mgr.34101) 1138 : cluster 0 pgmap v1859: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:55.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:54 smithi195 bash[66542]: audit 2024-03-07T03:51:54.282350+0000 mon.a (mon.0) 1978 : audit 0 from='client.? 172.21.15.148:0/427565064' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-07T03:51:55.561 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2817498546495 got 2817498546499 for osd.4 2024-03-07T03:51:55.561 DEBUG:teuthology.parallel:result is None 2024-03-07T03:51:57.177 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:56 smithi148 bash[79819]: cluster 2024-03-07T03:51:55.298548+0000 mgr.y (mgr.34101) 1139 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:57.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:56 smithi148 bash[82222]: cluster 2024-03-07T03:51:55.298548+0000 mgr.y (mgr.34101) 1139 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:57.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:56 smithi195 bash[66542]: cluster 2024-03-07T03:51:55.298548+0000 mgr.y (mgr.34101) 1139 : cluster 0 pgmap v1860: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:57.635 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:51:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:51:57] "GET /metrics HTTP/1.1" 200 34306 "" "Prometheus/2.43.0" 2024-03-07T03:51:58.212 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:57 smithi148 bash[79819]: audit 2024-03-07T03:51:57.355886+0000 mon.a (mon.0) 1979 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:58.213 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:57 smithi148 bash[82222]: audit 2024-03-07T03:51:57.355886+0000 mon.a (mon.0) 1979 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:58.214 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2186138354026 got 2186138354029 for osd.1 2024-03-07T03:51:58.214 DEBUG:teuthology.parallel:result is None 2024-03-07T03:51:58.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:57 smithi195 bash[66542]: audit 2024-03-07T03:51:57.355886+0000 mon.a (mon.0) 1979 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:51:58.884 INFO:teuthology.orchestra.run.smithi148.stdout:167503725096 2024-03-07T03:51:59.271 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:58 smithi148 bash[82222]: cluster 2024-03-07T03:51:57.299419+0000 mgr.y (mgr.34101) 1140 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:59.271 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:58 smithi148 bash[79819]: cluster 2024-03-07T03:51:57.299419+0000 mgr.y (mgr.34101) 1140 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:59.318 INFO:teuthology.orchestra.run.smithi148.stdout:236223201811 2024-03-07T03:51:59.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:58 smithi195 bash[66542]: cluster 2024-03-07T03:51:57.299419+0000 mgr.y (mgr.34101) 1140 : cluster 0 pgmap v1861: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:51:59.377 INFO:teuthology.orchestra.run.smithi148.stdout:2796023710026 2024-03-07T03:51:59.643 INFO:teuthology.orchestra.run.smithi148.stdout:2052994367860 2024-03-07T03:51:59.779 INFO:teuthology.orchestra.run.smithi148.stdout:201863463454 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[79819]: audit 2024-03-07T03:51:58.886486+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.148:0/2169501555' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[79819]: audit 2024-03-07T03:51:59.320503+0000 mon.a (mon.0) 1980 : audit 0 from='client.? 172.21.15.148:0/2516672876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[79819]: audit 2024-03-07T03:51:59.379811+0000 mon.a (mon.0) 1981 : audit 0 from='client.? 172.21.15.148:0/1347111710' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[79819]: audit 2024-03-07T03:51:59.645921+0000 mon.a (mon.0) 1982 : audit 0 from='client.? 172.21.15.148:0/3173679726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[79819]: audit 2024-03-07T03:51:59.782361+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.148:0/2008597986' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[82222]: audit 2024-03-07T03:51:58.886486+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.148:0/2169501555' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[82222]: audit 2024-03-07T03:51:59.320503+0000 mon.a (mon.0) 1980 : audit 0 from='client.? 172.21.15.148:0/2516672876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:51:59.950 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[82222]: audit 2024-03-07T03:51:59.379811+0000 mon.a (mon.0) 1981 : audit 0 from='client.? 172.21.15.148:0/1347111710' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:51:59.951 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[82222]: audit 2024-03-07T03:51:59.645921+0000 mon.a (mon.0) 1982 : audit 0 from='client.? 172.21.15.148:0/3173679726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:51:59.951 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:51:59 smithi148 bash[82222]: audit 2024-03-07T03:51:59.782361+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.148:0/2008597986' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:52:00.031 INFO:teuthology.orchestra.run.smithi148.stdout:2723009266003 2024-03-07T03:52:00.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:59 smithi195 bash[66542]: audit 2024-03-07T03:51:58.886486+0000 mon.c (mon.2) 84 : audit 0 from='client.? 172.21.15.148:0/2169501555' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-07T03:52:00.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:59 smithi195 bash[66542]: audit 2024-03-07T03:51:59.320503+0000 mon.a (mon.0) 1980 : audit 0 from='client.? 172.21.15.148:0/2516672876' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-07T03:52:00.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:59 smithi195 bash[66542]: audit 2024-03-07T03:51:59.379811+0000 mon.a (mon.0) 1981 : audit 0 from='client.? 172.21.15.148:0/1347111710' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-07T03:52:00.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:59 smithi195 bash[66542]: audit 2024-03-07T03:51:59.645921+0000 mon.a (mon.0) 1982 : audit 0 from='client.? 172.21.15.148:0/3173679726' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-07T03:52:00.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:51:59 smithi195 bash[66542]: audit 2024-03-07T03:51:59.782361+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.148:0/2008597986' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-07T03:52:01.126 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:00 smithi148 bash[79819]: cluster 2024-03-07T03:51:59.299954+0000 mgr.y (mgr.34101) 1141 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:01.126 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:00 smithi148 bash[79819]: audit 2024-03-07T03:52:00.033562+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.148:0/2820904951' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:52:01.127 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:00 smithi148 bash[82222]: cluster 2024-03-07T03:51:59.299954+0000 mgr.y (mgr.34101) 1141 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:01.127 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:00 smithi148 bash[82222]: audit 2024-03-07T03:52:00.033562+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.148:0/2820904951' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:52:01.128 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725092 got 167503725096 for osd.5 2024-03-07T03:52:01.128 DEBUG:teuthology.parallel:result is None 2024-03-07T03:52:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:00 smithi195 bash[66542]: cluster 2024-03-07T03:51:59.299954+0000 mgr.y (mgr.34101) 1141 : cluster 0 pgmap v1862: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:01.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:00 smithi195 bash[66542]: audit 2024-03-07T03:52:00.033562+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.148:0/2820904951' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-07T03:52:02.015 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2796023710022 got 2796023710026 for osd.3 2024-03-07T03:52:02.015 DEBUG:teuthology.parallel:result is None 2024-03-07T03:52:02.135 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201807 got 236223201811 for osd.7 2024-03-07T03:52:02.135 DEBUG:teuthology.parallel:result is None 2024-03-07T03:52:02.148 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2052994367856 got 2052994367860 for osd.0 2024-03-07T03:52:02.149 DEBUG:teuthology.parallel:result is None 2024-03-07T03:52:02.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:52:02 smithi195 bash[59134]: ts=2024-03-07T03:52:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:02.938 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463450 got 201863463454 for osd.6 2024-03-07T03:52:02.938 DEBUG:teuthology.parallel:result is None 2024-03-07T03:52:02.954 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2723009265999 got 2723009266003 for osd.2 2024-03-07T03:52:02.954 DEBUG:teuthology.parallel:result is None 2024-03-07T03:52:02.954 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-07T03:52:02.954 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:52:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:02 smithi148 bash[79819]: cluster 2024-03-07T03:52:01.300814+0000 mgr.y (mgr.34101) 1142 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:03.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:02 smithi148 bash[82222]: cluster 2024-03-07T03:52:01.300814+0000 mgr.y (mgr.34101) 1142 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:03.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:02 smithi195 bash[66542]: cluster 2024-03-07T03:52:01.300814+0000 mgr.y (mgr.34101) 1142 : cluster 0 pgmap v1863: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:05.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:04 smithi148 bash[79819]: cluster 2024-03-07T03:52:03.301616+0000 mgr.y (mgr.34101) 1143 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:05.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:04 smithi148 bash[82222]: cluster 2024-03-07T03:52:03.301616+0000 mgr.y (mgr.34101) 1143 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:05.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:04 smithi195 bash[66542]: cluster 2024-03-07T03:52:03.301616+0000 mgr.y (mgr.34101) 1143 : cluster 0 pgmap v1864: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:07.177 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:06 smithi148 bash[79819]: cluster 2024-03-07T03:52:05.302597+0000 mgr.y (mgr.34101) 1144 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:07.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:06 smithi148 bash[82222]: cluster 2024-03-07T03:52:05.302597+0000 mgr.y (mgr.34101) 1144 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:07.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:06 smithi195 bash[66542]: cluster 2024-03-07T03:52:05.302597+0000 mgr.y (mgr.34101) 1144 : cluster 0 pgmap v1865: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:07.551 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:52:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:52:07] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:52:07.727 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:08.909 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:08 smithi148 bash[79819]: cluster 2024-03-07T03:52:07.303343+0000 mgr.y (mgr.34101) 1145 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:08.909 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:08 smithi148 bash[82222]: cluster 2024-03-07T03:52:07.303343+0000 mgr.y (mgr.34101) 1145 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:09.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:08 smithi195 bash[66542]: cluster 2024-03-07T03:52:07.303343+0000 mgr.y (mgr.34101) 1145 : cluster 0 pgmap v1866: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:10.528 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:52:10.530 INFO:teuthology.orchestra.run.smithi148.stderr:dumped all 2024-03-07T03:52:11.218 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:10 smithi148 bash[79819]: cluster 2024-03-07T03:52:09.304429+0000 mgr.y (mgr.34101) 1146 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:11.218 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:10 smithi148 bash[82222]: cluster 2024-03-07T03:52:09.304429+0000 mgr.y (mgr.34101) 1146 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:11.219 INFO:teuthology.orchestra.run.smithi148.stdout:{"pg_ready":true,"pg_map":{"version":1867,"stamp":"2024-03-07T03:52:09.303638+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":565,"num_read_kb":992,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":721188,"kb_used_data":104652,"kb_used_omap":330,"kb_used_meta":616117,"kb_avail":749240028,"statfs":{"total":767960285184,"available":767221788672,"internally_reserved":0,"allocated":107163648,"data_stored":67557258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":338496,"internal_metadata":630904256},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004666"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":1509,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.822124+0000","last_change":"2024-03-07T03:25:05.767438+0000","last_active":"2024-03-07T03:50:51.822124+0000","last_peered":"2024-03-07T03:50:51.822124+0000","last_clean":"2024-03-07T03:50:51.822124+0000","last_became_active":"2024-03-07T03:25:05.766821+0000","last_became_peered":"2024-03-07T03:25:05.766821+0000","last_unstale":"2024-03-07T03:50:51.822124+0000","last_undegraded":"2024-03-07T03:50:51.822124+0000","last_fullsized":"2024-03-07T03:50:51.822124+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:22:24.522043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":1664,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.717103+0000","last_change":"2024-03-07T03:23:51.892673+0000","last_active":"2024-03-07T03:50:52.717103+0000","last_peered":"2024-03-07T03:50:52.717103+0000","last_clean":"2024-03-07T03:50:52.717103+0000","last_became_active":"2024-03-07T03:23:51.892162+0000","last_became_peered":"2024-03-07T03:23:51.892162+0000","last_unstale":"2024-03-07T03:50:52.717103+0000","last_undegraded":"2024-03-07T03:50:52.717103+0000","last_fullsized":"2024-03-07T03:50:52.717103+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:01:28.968065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136198+0000","last_change":"2024-03-07T03:24:36.552297+0000","last_active":"2024-03-07T03:50:52.136198+0000","last_peered":"2024-03-07T03:50:52.136198+0000","last_clean":"2024-03-07T03:50:52.136198+0000","last_became_active":"2024-03-07T03:24:36.549174+0000","last_became_peered":"2024-03-07T03:24:36.549174+0000","last_unstale":"2024-03-07T03:50:52.136198+0000","last_undegraded":"2024-03-07T03:50:52.136198+0000","last_fullsized":"2024-03-07T03:50:52.136198+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:27:11.217391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":1520,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138535+0000","last_change":"2024-03-07T03:24:36.594350+0000","last_active":"2024-03-07T03:50:52.138535+0000","last_peered":"2024-03-07T03:50:52.138535+0000","last_clean":"2024-03-07T03:50:52.138535+0000","last_became_active":"2024-03-07T03:24:36.583103+0000","last_became_peered":"2024-03-07T03:24:36.583103+0000","last_unstale":"2024-03-07T03:50:52.138535+0000","last_undegraded":"2024-03-07T03:50:52.138535+0000","last_fullsized":"2024-03-07T03:50:52.138535+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:53:33.397673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1540,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.824723+0000","last_change":"2024-03-07T03:25:05.760335+0000","last_active":"2024-03-07T03:50:51.824723+0000","last_peered":"2024-03-07T03:50:51.824723+0000","last_clean":"2024-03-07T03:50:51.824723+0000","last_became_active":"2024-03-07T03:25:05.760015+0000","last_became_peered":"2024-03-07T03:25:05.760015+0000","last_unstale":"2024-03-07T03:50:51.824723+0000","last_undegraded":"2024-03-07T03:50:51.824723+0000","last_fullsized":"2024-03-07T03:50:51.824723+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:58:03.505696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":1066,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828741+0000","last_change":"2024-03-07T03:25:05.754713+0000","last_active":"2024-03-07T03:50:51.828741+0000","last_peered":"2024-03-07T03:50:51.828741+0000","last_clean":"2024-03-07T03:50:51.828741+0000","last_became_active":"2024-03-07T03:25:05.753694+0000","last_became_peered":"2024-03-07T03:25:05.753694+0000","last_unstale":"2024-03-07T03:50:51.828741+0000","last_undegraded":"2024-03-07T03:50:51.828741+0000","last_fullsized":"2024-03-07T03:50:51.828741+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:06:44.140987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1540,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823643+0000","last_change":"2024-03-07T03:25:05.766773+0000","last_active":"2024-03-07T03:50:51.823643+0000","last_peered":"2024-03-07T03:50:51.823643+0000","last_clean":"2024-03-07T03:50:51.823643+0000","last_became_active":"2024-03-07T03:25:05.766466+0000","last_became_peered":"2024-03-07T03:25:05.766466+0000","last_unstale":"2024-03-07T03:50:51.823643+0000","last_undegraded":"2024-03-07T03:50:51.823643+0000","last_fullsized":"2024-03-07T03:50:51.823643+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:36:38.372426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":1081,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.826362+0000","last_change":"2024-03-07T03:25:05.755186+0000","last_active":"2024-03-07T03:50:51.826362+0000","last_peered":"2024-03-07T03:50:51.826362+0000","last_clean":"2024-03-07T03:50:51.826362+0000","last_became_active":"2024-03-07T03:25:05.754809+0000","last_became_peered":"2024-03-07T03:25:05.754809+0000","last_unstale":"2024-03-07T03:50:51.826362+0000","last_undegraded":"2024-03-07T03:50:51.826362+0000","last_fullsized":"2024-03-07T03:50:51.826362+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:17:44.629246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1510,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828064+0000","last_change":"2024-03-07T03:24:36.577321+0000","last_active":"2024-03-07T03:50:51.828064+0000","last_peered":"2024-03-07T03:50:51.828064+0000","last_clean":"2024-03-07T03:50:51.828064+0000","last_became_active":"2024-03-07T03:24:36.576821+0000","last_became_peered":"2024-03-07T03:24:36.576821+0000","last_unstale":"2024-03-07T03:50:51.828064+0000","last_undegraded":"2024-03-07T03:50:51.828064+0000","last_fullsized":"2024-03-07T03:50:51.828064+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:22:15.850424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":1656,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712752+0000","last_change":"2024-03-07T03:25:05.760637+0000","last_active":"2024-03-07T03:50:52.712752+0000","last_peered":"2024-03-07T03:50:52.712752+0000","last_clean":"2024-03-07T03:50:52.712752+0000","last_became_active":"2024-03-07T03:25:05.759731+0000","last_became_peered":"2024-03-07T03:25:05.759731+0000","last_unstale":"2024-03-07T03:50:52.712752+0000","last_undegraded":"2024-03-07T03:50:52.712752+0000","last_fullsized":"2024-03-07T03:50:52.712752+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:37:00.229672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1505,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.820386+0000","last_change":"2024-03-07T03:25:05.758930+0000","last_active":"2024-03-07T03:50:51.820386+0000","last_peered":"2024-03-07T03:50:51.820386+0000","last_clean":"2024-03-07T03:50:51.820386+0000","last_became_active":"2024-03-07T03:25:05.752951+0000","last_became_peered":"2024-03-07T03:25:05.752951+0000","last_unstale":"2024-03-07T03:50:51.820386+0000","last_undegraded":"2024-03-07T03:50:51.820386+0000","last_fullsized":"2024-03-07T03:50:51.820386+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:57:46.946958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":1667,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.711863+0000","last_change":"2024-03-07T03:25:05.756613+0000","last_active":"2024-03-07T03:50:52.711863+0000","last_peered":"2024-03-07T03:50:52.711863+0000","last_clean":"2024-03-07T03:50:52.711863+0000","last_became_active":"2024-03-07T03:25:05.755569+0000","last_became_peered":"2024-03-07T03:25:05.755569+0000","last_unstale":"2024-03-07T03:50:52.711863+0000","last_undegraded":"2024-03-07T03:50:52.711863+0000","last_fullsized":"2024-03-07T03:50:52.711863+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:53:55.456725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1075,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.037505+0000","last_change":"2024-03-07T03:23:51.896947+0000","last_active":"2024-03-07T03:50:48.037505+0000","last_peered":"2024-03-07T03:50:48.037505+0000","last_clean":"2024-03-07T03:50:48.037505+0000","last_became_active":"2024-03-07T03:23:51.894905+0000","last_became_peered":"2024-03-07T03:23:51.894905+0000","last_unstale":"2024-03-07T03:50:48.037505+0000","last_undegraded":"2024-03-07T03:50:48.037505+0000","last_fullsized":"2024-03-07T03:50:48.037505+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:53:19.480650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1526,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136673+0000","last_change":"2024-03-07T03:24:36.563029+0000","last_active":"2024-03-07T03:50:52.136673+0000","last_peered":"2024-03-07T03:50:52.136673+0000","last_clean":"2024-03-07T03:50:52.136673+0000","last_became_active":"2024-03-07T03:24:36.551321+0000","last_became_peered":"2024-03-07T03:24:36.551321+0000","last_unstale":"2024-03-07T03:50:52.136673+0000","last_undegraded":"2024-03-07T03:50:52.136673+0000","last_fullsized":"2024-03-07T03:50:52.136673+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:24:02.449683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1638,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.725514+0000","last_change":"2024-03-07T03:25:05.745431+0000","last_active":"2024-03-07T03:50:52.725514+0000","last_peered":"2024-03-07T03:50:52.725514+0000","last_clean":"2024-03-07T03:50:52.725514+0000","last_became_active":"2024-03-07T03:25:05.743847+0000","last_became_peered":"2024-03-07T03:25:05.743847+0000","last_unstale":"2024-03-07T03:50:52.725514+0000","last_undegraded":"2024-03-07T03:50:52.725514+0000","last_fullsized":"2024-03-07T03:50:52.725514+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:49:40.151264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138676+0000","last_change":"2024-03-07T03:24:36.589494+0000","last_active":"2024-03-07T03:50:52.138676+0000","last_peered":"2024-03-07T03:50:52.138676+0000","last_clean":"2024-03-07T03:50:52.138676+0000","last_became_active":"2024-03-07T03:24:36.575936+0000","last_became_peered":"2024-03-07T03:24:36.575936+0000","last_unstale":"2024-03-07T03:50:52.138676+0000","last_undegraded":"2024-03-07T03:50:52.138676+0000","last_fullsized":"2024-03-07T03:50:52.138676+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:22:21.229952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1645,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712342+0000","last_change":"2024-03-07T03:21:07.226408+0000","last_active":"2024-03-07T03:50:52.712342+0000","last_peered":"2024-03-07T03:50:52.712342+0000","last_clean":"2024-03-07T03:50:52.712342+0000","last_became_active":"2024-03-07T03:21:07.222660+0000","last_became_peered":"2024-03-07T03:21:07.222660+0000","last_unstale":"2024-03-07T03:50:52.712342+0000","last_undegraded":"2024-03-07T03:50:52.712342+0000","last_fullsized":"2024-03-07T03:50:52.712342+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:57:31.735684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1636,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.719964+0000","last_change":"2024-03-07T03:23:51.867358+0000","last_active":"2024-03-07T03:50:52.719964+0000","last_peered":"2024-03-07T03:50:52.719964+0000","last_clean":"2024-03-07T03:50:52.719964+0000","last_became_active":"2024-03-07T03:23:51.866212+0000","last_became_peered":"2024-03-07T03:23:51.866212+0000","last_unstale":"2024-03-07T03:50:52.719964+0000","last_undegraded":"2024-03-07T03:50:52.719964+0000","last_fullsized":"2024-03-07T03:50:52.719964+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:10:54.727645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1091,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.717966+0000","last_change":"2024-03-07T03:25:05.742408+0000","last_active":"2024-03-07T03:50:52.717966+0000","last_peered":"2024-03-07T03:50:52.717966+0000","last_clean":"2024-03-07T03:50:52.717966+0000","last_became_active":"2024-03-07T03:25:05.742000+0000","last_became_peered":"2024-03-07T03:25:05.742000+0000","last_unstale":"2024-03-07T03:50:52.717966+0000","last_undegraded":"2024-03-07T03:50:52.717966+0000","last_fullsized":"2024-03-07T03:50:52.717966+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:33:59.846812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1663,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.711585+0000","last_change":"2024-03-07T03:23:51.890859+0000","last_active":"2024-03-07T03:50:52.711585+0000","last_peered":"2024-03-07T03:50:52.711585+0000","last_clean":"2024-03-07T03:50:52.711585+0000","last_became_active":"2024-03-07T03:23:51.889934+0000","last_became_peered":"2024-03-07T03:23:51.889934+0000","last_unstale":"2024-03-07T03:50:52.711585+0000","last_undegraded":"2024-03-07T03:50:52.711585+0000","last_fullsized":"2024-03-07T03:50:52.711585+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:48:28.789581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1663,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712302+0000","last_change":"2024-03-07T03:25:05.761992+0000","last_active":"2024-03-07T03:50:52.712302+0000","last_peered":"2024-03-07T03:50:52.712302+0000","last_clean":"2024-03-07T03:50:52.712302+0000","last_became_active":"2024-03-07T03:25:05.761137+0000","last_became_peered":"2024-03-07T03:25:05.761137+0000","last_unstale":"2024-03-07T03:50:52.712302+0000","last_undegraded":"2024-03-07T03:50:52.712302+0000","last_fullsized":"2024-03-07T03:50:52.712302+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:45:27.321398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1092,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712356+0000","last_change":"2024-03-07T03:21:38.181209+0000","last_active":"2024-03-07T03:50:52.712356+0000","last_peered":"2024-03-07T03:50:52.712356+0000","last_clean":"2024-03-07T03:50:52.712356+0000","last_became_active":"2024-03-07T03:21:38.180257+0000","last_became_peered":"2024-03-07T03:21:38.180257+0000","last_unstale":"2024-03-07T03:50:52.712356+0000","last_undegraded":"2024-03-07T03:50:52.712356+0000","last_fullsized":"2024-03-07T03:50:52.712356+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:04:43.623665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1070,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.722895+0000","last_change":"2024-03-07T03:25:05.745658+0000","last_active":"2024-03-07T03:50:52.722895+0000","last_peered":"2024-03-07T03:50:52.722895+0000","last_clean":"2024-03-07T03:50:52.722895+0000","last_became_active":"2024-03-07T03:25:05.745333+0000","last_became_peered":"2024-03-07T03:25:05.745333+0000","last_unstale":"2024-03-07T03:50:52.722895+0000","last_undegraded":"2024-03-07T03:50:52.722895+0000","last_fullsized":"2024-03-07T03:50:52.722895+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:50:08.866643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1509,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.135970+0000","last_change":"2024-03-07T03:24:36.587414+0000","last_active":"2024-03-07T03:50:52.135970+0000","last_peered":"2024-03-07T03:50:52.135970+0000","last_clean":"2024-03-07T03:50:52.135970+0000","last_became_active":"2024-03-07T03:24:36.580492+0000","last_became_peered":"2024-03-07T03:24:36.580492+0000","last_unstale":"2024-03-07T03:50:52.135970+0000","last_undegraded":"2024-03-07T03:50:52.135970+0000","last_fullsized":"2024-03-07T03:50:52.135970+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:14:27.280098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"288'161","reported_seq":1441,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:47.857521+0000","last_change":"2024-03-07T03:21:07.213336+0000","last_active":"2024-03-07T03:50:47.857521+0000","last_peered":"2024-03-07T03:50:47.857521+0000","last_clean":"2024-03-07T03:50:47.857521+0000","last_became_active":"2024-03-07T03:21:07.212427+0000","last_became_peered":"2024-03-07T03:21:07.212427+0000","last_unstale":"2024-03-07T03:50:47.857521+0000","last_undegraded":"2024-03-07T03:50:47.857521+0000","last_fullsized":"2024-03-07T03:50:47.857521+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_clean_scrub_stamp":"2024-03-07T03:03:44.169894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:42.812513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1070,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.826690+0000","last_change":"2024-03-07T03:23:51.895364+0000","last_active":"2024-03-07T03:50:51.826690+0000","last_peered":"2024-03-07T03:50:51.826690+0000","last_clean":"2024-03-07T03:50:51.826690+0000","last_became_active":"2024-03-07T03:23:51.894115+0000","last_became_peered":"2024-03-07T03:23:51.894115+0000","last_unstale":"2024-03-07T03:50:51.826690+0000","last_undegraded":"2024-03-07T03:50:51.826690+0000","last_fullsized":"2024-03-07T03:50:51.826690+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:39:22.323228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1644,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.718858+0000","last_change":"2024-03-07T03:21:38.217039+0000","last_active":"2024-03-07T03:50:52.718858+0000","last_peered":"2024-03-07T03:50:52.718858+0000","last_clean":"2024-03-07T03:50:52.718858+0000","last_became_active":"2024-03-07T03:21:38.208087+0000","last_became_peered":"2024-03-07T03:21:38.208087+0000","last_unstale":"2024-03-07T03:50:52.718858+0000","last_undegraded":"2024-03-07T03:50:52.718858+0000","last_fullsized":"2024-03-07T03:50:52.718858+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:27:20.515324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1638,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.720182+0000","last_change":"2024-03-07T03:23:51.866591+0000","last_active":"2024-03-07T03:50:52.720182+0000","last_peered":"2024-03-07T03:50:52.720182+0000","last_clean":"2024-03-07T03:50:52.720182+0000","last_became_active":"2024-03-07T03:23:51.866176+0000","last_became_peered":"2024-03-07T03:23:51.866176+0000","last_unstale":"2024-03-07T03:50:52.720182+0000","last_undegraded":"2024-03-07T03:50:52.720182+0000","last_fullsized":"2024-03-07T03:50:52.720182+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:17:18.757845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.825469+0000","last_change":"2024-03-07T03:24:36.575901+0000","last_active":"2024-03-07T03:50:51.825469+0000","last_peered":"2024-03-07T03:50:51.825469+0000","last_clean":"2024-03-07T03:50:51.825469+0000","last_became_active":"2024-03-07T03:24:36.575414+0000","last_became_peered":"2024-03-07T03:24:36.575414+0000","last_unstale":"2024-03-07T03:50:51.825469+0000","last_undegraded":"2024-03-07T03:50:51.825469+0000","last_fullsized":"2024-03-07T03:50:51.825469+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:40:41.261906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1075,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.825378+0000","last_change":"2024-03-07T03:24:36.565742+0000","last_active":"2024-03-07T03:50:51.825378+0000","last_peered":"2024-03-07T03:50:51.825378+0000","last_clean":"2024-03-07T03:50:51.825378+0000","last_became_active":"2024-03-07T03:24:36.565423+0000","last_became_peered":"2024-03-07T03:24:36.565423+0000","last_unstale":"2024-03-07T03:50:51.825378+0000","last_undegraded":"2024-03-07T03:50:51.825378+0000","last_fullsized":"2024-03-07T03:50:51.825378+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:32:37.673607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1523,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.822876+0000","last_change":"2024-03-07T03:25:05.750215+0000","last_active":"2024-03-07T03:50:51.822876+0000","last_peered":"2024-03-07T03:50:51.822876+0000","last_clean":"2024-03-07T03:50:51.822876+0000","last_became_active":"2024-03-07T03:25:05.749173+0000","last_became_peered":"2024-03-07T03:25:05.749173+0000","last_unstale":"2024-03-07T03:50:51.822876+0000","last_undegraded":"2024-03-07T03:50:51.822876+0000","last_fullsized":"2024-03-07T03:50:51.822876+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:51:20.791266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1070,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823740+0000","last_change":"2024-03-07T03:21:38.190092+0000","last_active":"2024-03-07T03:50:51.823740+0000","last_peered":"2024-03-07T03:50:51.823740+0000","last_clean":"2024-03-07T03:50:51.823740+0000","last_became_active":"2024-03-07T03:21:38.189299+0000","last_became_peered":"2024-03-07T03:21:38.189299+0000","last_unstale":"2024-03-07T03:50:51.823740+0000","last_undegraded":"2024-03-07T03:50:51.823740+0000","last_fullsized":"2024-03-07T03:50:51.823740+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:32:45.193894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1637,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714919+0000","last_change":"2024-03-07T03:25:05.745377+0000","last_active":"2024-03-07T03:50:52.714919+0000","last_peered":"2024-03-07T03:50:52.714919+0000","last_clean":"2024-03-07T03:50:52.714919+0000","last_became_active":"2024-03-07T03:25:05.742616+0000","last_became_peered":"2024-03-07T03:25:05.742616+0000","last_unstale":"2024-03-07T03:50:52.714919+0000","last_undegraded":"2024-03-07T03:50:52.714919+0000","last_fullsized":"2024-03-07T03:50:52.714919+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:33:44.200407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1080,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828225+0000","last_change":"2024-03-07T03:24:36.552105+0000","last_active":"2024-03-07T03:50:51.828225+0000","last_peered":"2024-03-07T03:50:51.828225+0000","last_clean":"2024-03-07T03:50:51.828225+0000","last_became_active":"2024-03-07T03:24:36.551714+0000","last_became_peered":"2024-03-07T03:24:36.551714+0000","last_unstale":"2024-03-07T03:50:51.828225+0000","last_undegraded":"2024-03-07T03:50:51.828225+0000","last_fullsized":"2024-03-07T03:50:51.828225+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:05:39.898596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1649,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.720322+0000","last_change":"2024-03-07T03:21:38.217877+0000","last_active":"2024-03-07T03:50:52.720322+0000","last_peered":"2024-03-07T03:50:52.720322+0000","last_clean":"2024-03-07T03:50:52.720322+0000","last_became_active":"2024-03-07T03:21:38.217589+0000","last_became_peered":"2024-03-07T03:21:38.217589+0000","last_unstale":"2024-03-07T03:50:52.720322+0000","last_undegraded":"2024-03-07T03:50:52.720322+0000","last_fullsized":"2024-03-07T03:50:52.720322+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:56:05.365717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1645,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.718470+0000","last_change":"2024-03-07T03:25:05.745198+0000","last_active":"2024-03-07T03:50:52.718470+0000","last_peered":"2024-03-07T03:50:52.718470+0000","last_clean":"2024-03-07T03:50:52.718470+0000","last_became_active":"2024-03-07T03:25:05.744778+0000","last_became_peered":"2024-03-07T03:25:05.744778+0000","last_unstale":"2024-03-07T03:50:52.718470+0000","last_undegraded":"2024-03-07T03:50:52.718470+0000","last_fullsized":"2024-03-07T03:50:52.718470+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:38:00.822111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1515,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136409+0000","last_change":"2024-03-07T03:24:36.569612+0000","last_active":"2024-03-07T03:50:52.136409+0000","last_peered":"2024-03-07T03:50:52.136409+0000","last_clean":"2024-03-07T03:50:52.136409+0000","last_became_active":"2024-03-07T03:24:36.554169+0000","last_became_peered":"2024-03-07T03:24:36.554169+0000","last_unstale":"2024-03-07T03:50:52.136409+0000","last_undegraded":"2024-03-07T03:50:52.136409+0000","last_fullsized":"2024-03-07T03:50:52.136409+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:20:48.944499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1510,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.830096+0000","last_change":"2024-03-07T03:25:05.766616+0000","last_active":"2024-03-07T03:50:51.830096+0000","last_peered":"2024-03-07T03:50:51.830096+0000","last_clean":"2024-03-07T03:50:51.830096+0000","last_became_active":"2024-03-07T03:25:05.766291+0000","last_became_peered":"2024-03-07T03:25:05.766291+0000","last_unstale":"2024-03-07T03:50:51.830096+0000","last_undegraded":"2024-03-07T03:50:51.830096+0000","last_fullsized":"2024-03-07T03:50:51.830096+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T15:03:52.746085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1512,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.821130+0000","last_change":"2024-03-07T03:25:05.732068+0000","last_active":"2024-03-07T03:50:51.821130+0000","last_peered":"2024-03-07T03:50:51.821130+0000","last_clean":"2024-03-07T03:50:51.821130+0000","last_became_active":"2024-03-07T03:25:05.731620+0000","last_became_peered":"2024-03-07T03:25:05.731620+0000","last_unstale":"2024-03-07T03:50:51.821130+0000","last_undegraded":"2024-03-07T03:50:51.821130+0000","last_fullsized":"2024-03-07T03:50:51.821130+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:34:43.559628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1520,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823998+0000","last_change":"2024-03-07T03:25:05.751860+0000","last_active":"2024-03-07T03:50:51.823998+0000","last_peered":"2024-03-07T03:50:51.823998+0000","last_clean":"2024-03-07T03:50:51.823998+0000","last_became_active":"2024-03-07T03:25:05.751351+0000","last_became_peered":"2024-03-07T03:25:05.751351+0000","last_unstale":"2024-03-07T03:50:51.823998+0000","last_undegraded":"2024-03-07T03:50:51.823998+0000","last_fullsized":"2024-03-07T03:50:51.823998+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:17:13.168268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1085,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.039591+0000","last_change":"2024-03-07T03:25:05.740806+0000","last_active":"2024-03-07T03:50:48.039591+0000","last_peered":"2024-03-07T03:50:48.039591+0000","last_clean":"2024-03-07T03:50:48.039591+0000","last_became_active":"2024-03-07T03:25:05.740439+0000","last_became_peered":"2024-03-07T03:25:05.740439+0000","last_unstale":"2024-03-07T03:50:48.039591+0000","last_undegraded":"2024-03-07T03:50:48.039591+0000","last_fullsized":"2024-03-07T03:50:48.039591+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:29:36.069032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1527,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.827864+0000","last_change":"2024-03-07T03:25:05.764961+0000","last_active":"2024-03-07T03:50:51.827864+0000","last_peered":"2024-03-07T03:50:51.827864+0000","last_clean":"2024-03-07T03:50:51.827864+0000","last_became_active":"2024-03-07T03:25:05.764581+0000","last_became_peered":"2024-03-07T03:25:05.764581+0000","last_unstale":"2024-03-07T03:50:51.827864+0000","last_undegraded":"2024-03-07T03:50:51.827864+0000","last_fullsized":"2024-03-07T03:50:51.827864+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:44:56.467749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":1538,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136944+0000","last_change":"2024-03-07T03:25:05.741820+0000","last_active":"2024-03-07T03:50:52.136944+0000","last_peered":"2024-03-07T03:50:52.136944+0000","last_clean":"2024-03-07T03:50:52.136944+0000","last_became_active":"2024-03-07T03:25:05.741131+0000","last_became_peered":"2024-03-07T03:25:05.741131+0000","last_unstale":"2024-03-07T03:50:52.136944+0000","last_undegraded":"2024-03-07T03:50:52.136944+0000","last_fullsized":"2024-03-07T03:50:52.136944+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:53:27.540203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1662,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.711885+0000","last_change":"2024-03-07T03:25:05.760704+0000","last_active":"2024-03-07T03:50:52.711885+0000","last_peered":"2024-03-07T03:50:52.711885+0000","last_clean":"2024-03-07T03:50:52.711885+0000","last_became_active":"2024-03-07T03:25:05.759959+0000","last_became_peered":"2024-03-07T03:25:05.759959+0000","last_unstale":"2024-03-07T03:50:52.711885+0000","last_undegraded":"2024-03-07T03:50:52.711885+0000","last_fullsized":"2024-03-07T03:50:52.711885+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:14:32.135310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1064,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.825809+0000","last_change":"2024-03-07T03:24:36.562591+0000","last_active":"2024-03-07T03:50:51.825809+0000","last_peered":"2024-03-07T03:50:51.825809+0000","last_clean":"2024-03-07T03:50:51.825809+0000","last_became_active":"2024-03-07T03:24:36.561705+0000","last_became_peered":"2024-03-07T03:24:36.561705+0000","last_unstale":"2024-03-07T03:50:51.825809+0000","last_undegraded":"2024-03-07T03:50:51.825809+0000","last_fullsized":"2024-03-07T03:50:51.825809+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:10:00.333229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1078,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.037225+0000","last_change":"2024-03-07T03:23:51.872075+0000","last_active":"2024-03-07T03:50:48.037225+0000","last_peered":"2024-03-07T03:50:48.037225+0000","last_clean":"2024-03-07T03:50:48.037225+0000","last_became_active":"2024-03-07T03:23:51.869090+0000","last_became_peered":"2024-03-07T03:23:51.869090+0000","last_unstale":"2024-03-07T03:50:48.037225+0000","last_undegraded":"2024-03-07T03:50:48.037225+0000","last_fullsized":"2024-03-07T03:50:48.037225+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:56:43.404841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":1057,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.718105+0000","last_change":"2024-03-07T03:11:17.738764+0000","last_active":"2024-03-07T03:50:52.718105+0000","last_peered":"2024-03-07T03:50:52.718105+0000","last_clean":"2024-03-07T03:50:52.718105+0000","last_became_active":"2024-03-07T03:11:17.738151+0000","last_became_peered":"2024-03-07T03:11:17.738151+0000","last_unstale":"2024-03-07T03:50:52.718105+0000","last_undegraded":"2024-03-07T03:50:52.718105+0000","last_fullsized":"2024-03-07T03:50:52.718105+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:37:22.289110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1543,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.820673+0000","last_change":"2024-03-07T03:25:05.769855+0000","last_active":"2024-03-07T03:50:51.820673+0000","last_peered":"2024-03-07T03:50:51.820673+0000","last_clean":"2024-03-07T03:50:51.820673+0000","last_became_active":"2024-03-07T03:25:05.768412+0000","last_became_peered":"2024-03-07T03:25:05.768412+0000","last_unstale":"2024-03-07T03:50:51.820673+0000","last_undegraded":"2024-03-07T03:50:51.820673+0000","last_fullsized":"2024-03-07T03:50:51.820673+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:29:23.223181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":1089,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.713748+0000","last_change":"2024-03-07T03:25:05.740687+0000","last_active":"2024-03-07T03:50:52.713748+0000","last_peered":"2024-03-07T03:50:52.713748+0000","last_clean":"2024-03-07T03:50:52.713748+0000","last_became_active":"2024-03-07T03:25:05.738724+0000","last_became_peered":"2024-03-07T03:25:05.738724+0000","last_unstale":"2024-03-07T03:50:52.713748+0000","last_undegraded":"2024-03-07T03:50:52.713748+0000","last_fullsized":"2024-03-07T03:50:52.713748+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:13:31.913269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1048,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.037894+0000","last_change":"2024-03-07T03:11:17.846593+0000","last_active":"2024-03-07T03:50:48.037894+0000","last_peered":"2024-03-07T03:50:48.037894+0000","last_clean":"2024-03-07T03:50:48.037894+0000","last_became_active":"2024-03-07T03:11:17.845734+0000","last_became_peered":"2024-03-07T03:11:17.845734+0000","last_unstale":"2024-03-07T03:50:48.037894+0000","last_undegraded":"2024-03-07T03:50:48.037894+0000","last_fullsized":"2024-03-07T03:50:48.037894+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:18:40.161672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":1086,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.717325+0000","last_change":"2024-03-07T03:25:05.739761+0000","last_active":"2024-03-07T03:50:52.717325+0000","last_peered":"2024-03-07T03:50:52.717325+0000","last_clean":"2024-03-07T03:50:52.717325+0000","last_became_active":"2024-03-07T03:25:05.739351+0000","last_became_peered":"2024-03-07T03:25:05.739351+0000","last_unstale":"2024-03-07T03:50:52.717325+0000","last_undegraded":"2024-03-07T03:50:52.717325+0000","last_fullsized":"2024-03-07T03:50:52.717325+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:34:39.300885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1513,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138608+0000","last_change":"2024-03-07T03:24:36.575581+0000","last_active":"2024-03-07T03:50:52.138608+0000","last_peered":"2024-03-07T03:50:52.138608+0000","last_clean":"2024-03-07T03:50:52.138608+0000","last_became_active":"2024-03-07T03:24:36.562442+0000","last_became_peered":"2024-03-07T03:24:36.562442+0000","last_unstale":"2024-03-07T03:50:52.138608+0000","last_undegraded":"2024-03-07T03:50:52.138608+0000","last_fullsized":"2024-03-07T03:50:52.138608+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:27:06.200975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":1085,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712618+0000","last_change":"2024-03-07T03:21:07.224986+0000","last_active":"2024-03-07T03:50:52.712618+0000","last_peered":"2024-03-07T03:50:52.712618+0000","last_clean":"2024-03-07T03:50:52.712618+0000","last_became_active":"2024-03-07T03:21:07.223839+0000","last_became_peered":"2024-03-07T03:21:07.223839+0000","last_unstale":"2024-03-07T03:50:52.712618+0000","last_undegraded":"2024-03-07T03:50:52.712618+0000","last_fullsized":"2024-03-07T03:50:52.712618+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:29:44.461183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1651,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.709106+0000","last_change":"2024-03-07T03:24:36.586658+0000","last_active":"2024-03-07T03:50:52.709106+0000","last_peered":"2024-03-07T03:50:52.709106+0000","last_clean":"2024-03-07T03:50:52.709106+0000","last_became_active":"2024-03-07T03:24:36.586284+0000","last_became_peered":"2024-03-07T03:24:36.586284+0000","last_unstale":"2024-03-07T03:50:52.709106+0000","last_undegraded":"2024-03-07T03:50:52.709106+0000","last_fullsized":"2024-03-07T03:50:52.709106+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:16:06.802596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1069,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712287+0000","last_change":"2024-03-07T03:25:05.749052+0000","last_active":"2024-03-07T03:50:52.712287+0000","last_peered":"2024-03-07T03:50:52.712287+0000","last_clean":"2024-03-07T03:50:52.712287+0000","last_became_active":"2024-03-07T03:25:05.742801+0000","last_became_peered":"2024-03-07T03:25:05.742801+0000","last_unstale":"2024-03-07T03:50:52.712287+0000","last_undegraded":"2024-03-07T03:50:52.712287+0000","last_fullsized":"2024-03-07T03:50:52.712287+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:15:09.074490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1061,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.040280+0000","last_change":"2024-03-07T03:24:36.546759+0000","last_active":"2024-03-07T03:50:48.040280+0000","last_peered":"2024-03-07T03:50:48.040280+0000","last_clean":"2024-03-07T03:50:48.040280+0000","last_became_active":"2024-03-07T03:24:36.545949+0000","last_became_peered":"2024-03-07T03:24:36.545949+0000","last_unstale":"2024-03-07T03:50:48.040280+0000","last_undegraded":"2024-03-07T03:50:48.040280+0000","last_fullsized":"2024-03-07T03:50:48.040280+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:33:39.293881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1652,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.710625+0000","last_change":"2024-03-07T03:24:36.566360+0000","last_active":"2024-03-07T03:50:52.710625+0000","last_peered":"2024-03-07T03:50:52.710625+0000","last_clean":"2024-03-07T03:50:52.710625+0000","last_became_active":"2024-03-07T03:24:36.565657+0000","last_became_peered":"2024-03-07T03:24:36.565657+0000","last_unstale":"2024-03-07T03:50:52.710625+0000","last_undegraded":"2024-03-07T03:50:52.710625+0000","last_fullsized":"2024-03-07T03:50:52.710625+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:44:47.754341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1513,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138073+0000","last_change":"2024-03-07T03:24:36.580300+0000","last_active":"2024-03-07T03:50:52.138073+0000","last_peered":"2024-03-07T03:50:52.138073+0000","last_clean":"2024-03-07T03:50:52.138073+0000","last_became_active":"2024-03-07T03:24:36.560593+0000","last_became_peered":"2024-03-07T03:24:36.560593+0000","last_unstale":"2024-03-07T03:50:52.138073+0000","last_undegraded":"2024-03-07T03:50:52.138073+0000","last_fullsized":"2024-03-07T03:50:52.138073+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:37:18.825686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":1088,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714554+0000","last_change":"2024-03-07T03:25:05.742630+0000","last_active":"2024-03-07T03:50:52.714554+0000","last_peered":"2024-03-07T03:50:52.714554+0000","last_clean":"2024-03-07T03:50:52.714554+0000","last_became_active":"2024-03-07T03:25:05.740480+0000","last_became_peered":"2024-03-07T03:25:05.740480+0000","last_unstale":"2024-03-07T03:50:52.714554+0000","last_undegraded":"2024-03-07T03:50:52.714554+0000","last_fullsized":"2024-03-07T03:50:52.714554+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:56:21.846159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1509,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823033+0000","last_change":"2024-03-07T03:25:05.767010+0000","last_active":"2024-03-07T03:50:51.823033+0000","last_peered":"2024-03-07T03:50:51.823033+0000","last_clean":"2024-03-07T03:50:51.823033+0000","last_became_active":"2024-03-07T03:25:05.766702+0000","last_became_peered":"2024-03-07T03:25:05.766702+0000","last_unstale":"2024-03-07T03:50:51.823033+0000","last_undegraded":"2024-03-07T03:50:51.823033+0000","last_fullsized":"2024-03-07T03:50:51.823033+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:37:06.236762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"363'23","reported_seq":1542,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828022+0000","last_change":"2024-03-07T03:25:05.735450+0000","last_active":"2024-03-07T03:50:51.828022+0000","last_peered":"2024-03-07T03:50:51.828022+0000","last_clean":"2024-03-07T03:50:51.828022+0000","last_became_active":"2024-03-07T03:25:05.734655+0000","last_became_peered":"2024-03-07T03:25:05.734655+0000","last_unstale":"2024-03-07T03:50:51.828022+0000","last_undegraded":"2024-03-07T03:50:51.828022+0000","last_fullsized":"2024-03-07T03:50:51.828022+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:34:59.285647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1636,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714535+0000","last_change":"2024-03-07T03:23:51.863430+0000","last_active":"2024-03-07T03:50:52.714535+0000","last_peered":"2024-03-07T03:50:52.714535+0000","last_clean":"2024-03-07T03:50:52.714535+0000","last_became_active":"2024-03-07T03:23:51.863022+0000","last_became_peered":"2024-03-07T03:23:51.863022+0000","last_unstale":"2024-03-07T03:50:52.714535+0000","last_undegraded":"2024-03-07T03:50:52.714535+0000","last_fullsized":"2024-03-07T03:50:52.714535+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:59:32.196016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":1089,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:47.859250+0000","last_change":"2024-03-07T03:25:05.740150+0000","last_active":"2024-03-07T03:50:47.859250+0000","last_peered":"2024-03-07T03:50:47.859250+0000","last_clean":"2024-03-07T03:50:47.859250+0000","last_became_active":"2024-03-07T03:25:05.738746+0000","last_became_peered":"2024-03-07T03:25:05.738746+0000","last_unstale":"2024-03-07T03:50:47.859250+0000","last_undegraded":"2024-03-07T03:50:47.859250+0000","last_fullsized":"2024-03-07T03:50:47.859250+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:25:03.546350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1506,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136287+0000","last_change":"2024-03-07T03:24:36.560614+0000","last_active":"2024-03-07T03:50:52.136287+0000","last_peered":"2024-03-07T03:50:52.136287+0000","last_clean":"2024-03-07T03:50:52.136287+0000","last_became_active":"2024-03-07T03:24:36.558250+0000","last_became_peered":"2024-03-07T03:24:36.558250+0000","last_unstale":"2024-03-07T03:50:52.136287+0000","last_undegraded":"2024-03-07T03:50:52.136287+0000","last_fullsized":"2024-03-07T03:50:52.136287+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:16:30.561539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1062,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.040199+0000","last_change":"2024-03-07T03:21:38.198598+0000","last_active":"2024-03-07T03:50:48.040199+0000","last_peered":"2024-03-07T03:50:48.040199+0000","last_clean":"2024-03-07T03:50:48.040199+0000","last_became_active":"2024-03-07T03:21:38.197849+0000","last_became_peered":"2024-03-07T03:21:38.197849+0000","last_unstale":"2024-03-07T03:50:48.040199+0000","last_undegraded":"2024-03-07T03:50:48.040199+0000","last_fullsized":"2024-03-07T03:50:48.040199+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:20:02.544918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1513,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136552+0000","last_change":"2024-03-07T03:24:36.521766+0000","last_active":"2024-03-07T03:50:52.136552+0000","last_peered":"2024-03-07T03:50:52.136552+0000","last_clean":"2024-03-07T03:50:52.136552+0000","last_became_active":"2024-03-07T03:24:36.520227+0000","last_became_peered":"2024-03-07T03:24:36.520227+0000","last_unstale":"2024-03-07T03:50:52.136552+0000","last_undegraded":"2024-03-07T03:50:52.136552+0000","last_fullsized":"2024-03-07T03:50:52.136552+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:30:26.946634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1651,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.710801+0000","last_change":"2024-03-07T03:24:36.553047+0000","last_active":"2024-03-07T03:50:52.710801+0000","last_peered":"2024-03-07T03:50:52.710801+0000","last_clean":"2024-03-07T03:50:52.710801+0000","last_became_active":"2024-03-07T03:24:36.552690+0000","last_became_peered":"2024-03-07T03:24:36.552690+0000","last_unstale":"2024-03-07T03:50:52.710801+0000","last_undegraded":"2024-03-07T03:50:52.710801+0000","last_fullsized":"2024-03-07T03:50:52.710801+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:34:00.975763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1070,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.236001+0000","last_change":"2024-03-07T03:21:38.200454+0000","last_active":"2024-03-07T03:50:48.236001+0000","last_peered":"2024-03-07T03:50:48.236001+0000","last_clean":"2024-03-07T03:50:48.236001+0000","last_became_active":"2024-03-07T03:21:38.199536+0000","last_became_peered":"2024-03-07T03:21:38.199536+0000","last_unstale":"2024-03-07T03:50:48.236001+0000","last_undegraded":"2024-03-07T03:50:48.236001+0000","last_fullsized":"2024-03-07T03:50:48.236001+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:59:25.711174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136851+0000","last_change":"2024-03-07T03:24:36.560408+0000","last_active":"2024-03-07T03:50:52.136851+0000","last_peered":"2024-03-07T03:50:52.136851+0000","last_clean":"2024-03-07T03:50:52.136851+0000","last_became_active":"2024-03-07T03:24:36.548964+0000","last_became_peered":"2024-03-07T03:24:36.548964+0000","last_unstale":"2024-03-07T03:50:52.136851+0000","last_undegraded":"2024-03-07T03:50:52.136851+0000","last_fullsized":"2024-03-07T03:50:52.136851+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:40:11.900535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1655,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.709037+0000","last_change":"2024-03-07T03:24:36.587014+0000","last_active":"2024-03-07T03:50:52.709037+0000","last_peered":"2024-03-07T03:50:52.709037+0000","last_clean":"2024-03-07T03:50:52.709037+0000","last_became_active":"2024-03-07T03:24:36.586353+0000","last_became_peered":"2024-03-07T03:24:36.586353+0000","last_unstale":"2024-03-07T03:50:52.709037+0000","last_undegraded":"2024-03-07T03:50:52.709037+0000","last_fullsized":"2024-03-07T03:50:52.709037+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:19:28.432486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1644,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714054+0000","last_change":"2024-03-07T03:25:05.744751+0000","last_active":"2024-03-07T03:50:52.714054+0000","last_peered":"2024-03-07T03:50:52.714054+0000","last_clean":"2024-03-07T03:50:52.714054+0000","last_became_active":"2024-03-07T03:25:05.744404+0000","last_became_peered":"2024-03-07T03:25:05.744404+0000","last_unstale":"2024-03-07T03:50:52.714054+0000","last_undegraded":"2024-03-07T03:50:52.714054+0000","last_fullsized":"2024-03-07T03:50:52.714054+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:12:03.756012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1521,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138153+0000","last_change":"2024-03-07T03:25:05.743998+0000","last_active":"2024-03-07T03:50:52.138153+0000","last_peered":"2024-03-07T03:50:52.138153+0000","last_clean":"2024-03-07T03:50:52.138153+0000","last_became_active":"2024-03-07T03:25:05.741440+0000","last_became_peered":"2024-03-07T03:25:05.741440+0000","last_unstale":"2024-03-07T03:50:52.138153+0000","last_undegraded":"2024-03-07T03:50:52.138153+0000","last_fullsized":"2024-03-07T03:50:52.138153+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:16:57.069373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":1527,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.824354+0000","last_change":"2024-03-07T03:25:05.767463+0000","last_active":"2024-03-07T03:50:51.824354+0000","last_peered":"2024-03-07T03:50:51.824354+0000","last_clean":"2024-03-07T03:50:51.824354+0000","last_became_active":"2024-03-07T03:25:05.766939+0000","last_became_peered":"2024-03-07T03:25:05.766939+0000","last_unstale":"2024-03-07T03:50:51.824354+0000","last_undegraded":"2024-03-07T03:50:51.824354+0000","last_fullsized":"2024-03-07T03:50:51.824354+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:26:47.320990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":65,"num_read_kb":68,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3677,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201814,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112604,"kb_used_data":13460,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93632548,"statfs":{"total":95995035648,"available":95879729152,"internally_reserved":0,"allocated":13783040,"data_stored":8816519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":101513483},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463457,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":138836,"kb_used_data":13448,"kb_used_omap":1,"kb_used_meta":125374,"kb_avail":93606316,"statfs":{"total":95995035648,"available":95852867584,"internally_reserved":0,"allocated":13770752,"data_stored":8816373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":128383243},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":509,"seq":2186138354032,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76604,"kb_used_data":12852,"kb_used_omap":53,"kb_used_meta":63690,"kb_avail":93668548,"statfs":{"total":95995035648,"available":95916593152,"internally_reserved":0,"allocated":13160448,"data_stored":8221191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54606,"internal_metadata":65219250},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":478,"seq":2052994367862,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":85516,"kb_used_data":13440,"kb_used_omap":59,"kb_used_meta":72004,"kb_avail":93659636,"statfs":{"total":95995035648,"available":95907467264,"internally_reserved":0,"allocated":13762560,"data_stored":8812163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":60713,"internal_metadata":73732823},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":634,"seq":2723009266005,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":71304,"kb_used_data":12860,"kb_used_omap":73,"kb_used_meta":58358,"kb_avail":93673848,"statfs":{"total":95995035648,"available":95922020352,"internally_reserved":0,"allocated":13168640,"data_stored":8221807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":75685,"internal_metadata":59758683},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":651,"seq":2796023710029,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":89348,"kb_used_data":12856,"kb_used_omap":67,"kb_used_meta":76412,"kb_avail":93655804,"statfs":{"total":95995035648,"available":95903543296,"internally_reserved":0,"allocated":13164544,"data_stored":8221281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69425,"internal_metadata":78246095},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":656,"seq":2817498546503,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62808,"kb_used_data":12876,"kb_used_omap":71,"kb_used_meta":49848,"kb_avail":93682344,"statfs":{"total":95995035648,"available":95930720256,"internally_reserved":0,"allocated":13185024,"data_stored":8226093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72724,"internal_metadata":51045356},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725099,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84168,"kb_used_data":12860,"kb_used_omap":1,"kb_used_meta":71294,"kb_avail":93660984,"statfs":{"total":95995035648,"available":95908847616,"internally_reserved":0,"allocated":13168640,"data_stored":8221831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":73005323},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1504,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":899,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1274,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:52:11.224 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph pg dump --format=json 2024-03-07T03:52:11.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:10 smithi195 bash[66542]: cluster 2024-03-07T03:52:09.304429+0000 mgr.y (mgr.34101) 1146 : cluster 0 pgmap v1867: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:12.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:11 smithi148 bash[79819]: audit 2024-03-07T03:52:10.507711+0000 mgr.y (mgr.34101) 1147 : audit 0 from='client.44957 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:52:12.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:11 smithi148 bash[82222]: audit 2024-03-07T03:52:10.507711+0000 mgr.y (mgr.34101) 1147 : audit 0 from='client.44957 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:52:12.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:11 smithi195 bash[66542]: audit 2024-03-07T03:52:10.507711+0000 mgr.y (mgr.34101) 1147 : audit 0 from='client.44957 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:52:12.572 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:52:12 smithi195 bash[59134]: ts=2024-03-07T03:52:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:13.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:12 smithi148 bash[79819]: cluster 2024-03-07T03:52:11.304966+0000 mgr.y (mgr.34101) 1148 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:13.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:12 smithi148 bash[79819]: audit 2024-03-07T03:52:12.356484+0000 mon.a (mon.0) 1983 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:13.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:12 smithi148 bash[82222]: cluster 2024-03-07T03:52:11.304966+0000 mgr.y (mgr.34101) 1148 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:13.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:12 smithi148 bash[82222]: audit 2024-03-07T03:52:12.356484+0000 mon.a (mon.0) 1983 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:13.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:12 smithi195 bash[66542]: cluster 2024-03-07T03:52:11.304966+0000 mgr.y (mgr.34101) 1148 : cluster 0 pgmap v1868: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:13.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:12 smithi195 bash[66542]: audit 2024-03-07T03:52:12.356484+0000 mon.a (mon.0) 1983 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:15.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:14 smithi148 bash[79819]: cluster 2024-03-07T03:52:13.305752+0000 mgr.y (mgr.34101) 1149 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:15.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:14 smithi148 bash[82222]: cluster 2024-03-07T03:52:13.305752+0000 mgr.y (mgr.34101) 1149 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:15.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:14 smithi195 bash[66542]: cluster 2024-03-07T03:52:13.305752+0000 mgr.y (mgr.34101) 1149 : cluster 0 pgmap v1869: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:15.963 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:17.177 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:16 smithi148 bash[79819]: cluster 2024-03-07T03:52:15.306675+0000 mgr.y (mgr.34101) 1150 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:17.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:16 smithi148 bash[82222]: cluster 2024-03-07T03:52:15.306675+0000 mgr.y (mgr.34101) 1150 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:17.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:16 smithi195 bash[66542]: cluster 2024-03-07T03:52:15.306675+0000 mgr.y (mgr.34101) 1150 : cluster 0 pgmap v1870: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:17.551 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:52:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:52:17] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:52:18.875 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:52:18.877 INFO:teuthology.orchestra.run.smithi148.stderr:dumped all 2024-03-07T03:52:19.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:18 smithi148 bash[79819]: cluster 2024-03-07T03:52:17.307453+0000 mgr.y (mgr.34101) 1151 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:19.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:18 smithi148 bash[82222]: cluster 2024-03-07T03:52:17.307453+0000 mgr.y (mgr.34101) 1151 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:19.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:18 smithi195 bash[66542]: cluster 2024-03-07T03:52:17.307453+0000 mgr.y (mgr.34101) 1151 : cluster 0 pgmap v1871: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:19.666 INFO:teuthology.orchestra.run.smithi148.stdout:{"pg_ready":true,"pg_map":{"version":1871,"stamp":"2024-03-07T03:52:17.307005+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595922,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":565,"num_read_kb":992,"num_write":300,"num_write_kb":2446,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":222,"ondisk_log_size":222,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":721188,"kb_used_data":104652,"kb_used_omap":330,"kb_used_meta":616117,"kb_avail":749240028,"statfs":{"total":767960285184,"available":767221788672,"internally_reserved":0,"allocated":107163648,"data_stored":67557258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":338496,"internal_metadata":630904256},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005093"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":1509,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.822124+0000","last_change":"2024-03-07T03:25:05.767438+0000","last_active":"2024-03-07T03:50:51.822124+0000","last_peered":"2024-03-07T03:50:51.822124+0000","last_clean":"2024-03-07T03:50:51.822124+0000","last_became_active":"2024-03-07T03:25:05.766821+0000","last_became_peered":"2024-03-07T03:25:05.766821+0000","last_unstale":"2024-03-07T03:50:51.822124+0000","last_undegraded":"2024-03-07T03:50:51.822124+0000","last_fullsized":"2024-03-07T03:50:51.822124+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:22:24.522043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":1664,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.717103+0000","last_change":"2024-03-07T03:23:51.892673+0000","last_active":"2024-03-07T03:50:52.717103+0000","last_peered":"2024-03-07T03:50:52.717103+0000","last_clean":"2024-03-07T03:50:52.717103+0000","last_became_active":"2024-03-07T03:23:51.892162+0000","last_became_peered":"2024-03-07T03:23:51.892162+0000","last_unstale":"2024-03-07T03:50:52.717103+0000","last_undegraded":"2024-03-07T03:50:52.717103+0000","last_fullsized":"2024-03-07T03:50:52.717103+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:01:28.968065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136198+0000","last_change":"2024-03-07T03:24:36.552297+0000","last_active":"2024-03-07T03:50:52.136198+0000","last_peered":"2024-03-07T03:50:52.136198+0000","last_clean":"2024-03-07T03:50:52.136198+0000","last_became_active":"2024-03-07T03:24:36.549174+0000","last_became_peered":"2024-03-07T03:24:36.549174+0000","last_unstale":"2024-03-07T03:50:52.136198+0000","last_undegraded":"2024-03-07T03:50:52.136198+0000","last_fullsized":"2024-03-07T03:50:52.136198+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:27:11.217391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":1520,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138535+0000","last_change":"2024-03-07T03:24:36.594350+0000","last_active":"2024-03-07T03:50:52.138535+0000","last_peered":"2024-03-07T03:50:52.138535+0000","last_clean":"2024-03-07T03:50:52.138535+0000","last_became_active":"2024-03-07T03:24:36.583103+0000","last_became_peered":"2024-03-07T03:24:36.583103+0000","last_unstale":"2024-03-07T03:50:52.138535+0000","last_undegraded":"2024-03-07T03:50:52.138535+0000","last_fullsized":"2024-03-07T03:50:52.138535+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:53:33.397673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1540,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.824723+0000","last_change":"2024-03-07T03:25:05.760335+0000","last_active":"2024-03-07T03:50:51.824723+0000","last_peered":"2024-03-07T03:50:51.824723+0000","last_clean":"2024-03-07T03:50:51.824723+0000","last_became_active":"2024-03-07T03:25:05.760015+0000","last_became_peered":"2024-03-07T03:25:05.760015+0000","last_unstale":"2024-03-07T03:50:51.824723+0000","last_undegraded":"2024-03-07T03:50:51.824723+0000","last_fullsized":"2024-03-07T03:50:51.824723+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:58:03.505696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":1066,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828741+0000","last_change":"2024-03-07T03:25:05.754713+0000","last_active":"2024-03-07T03:50:51.828741+0000","last_peered":"2024-03-07T03:50:51.828741+0000","last_clean":"2024-03-07T03:50:51.828741+0000","last_became_active":"2024-03-07T03:25:05.753694+0000","last_became_peered":"2024-03-07T03:25:05.753694+0000","last_unstale":"2024-03-07T03:50:51.828741+0000","last_undegraded":"2024-03-07T03:50:51.828741+0000","last_fullsized":"2024-03-07T03:50:51.828741+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:06:44.140987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1540,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823643+0000","last_change":"2024-03-07T03:25:05.766773+0000","last_active":"2024-03-07T03:50:51.823643+0000","last_peered":"2024-03-07T03:50:51.823643+0000","last_clean":"2024-03-07T03:50:51.823643+0000","last_became_active":"2024-03-07T03:25:05.766466+0000","last_became_peered":"2024-03-07T03:25:05.766466+0000","last_unstale":"2024-03-07T03:50:51.823643+0000","last_undegraded":"2024-03-07T03:50:51.823643+0000","last_fullsized":"2024-03-07T03:50:51.823643+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:36:38.372426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":1081,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.826362+0000","last_change":"2024-03-07T03:25:05.755186+0000","last_active":"2024-03-07T03:50:51.826362+0000","last_peered":"2024-03-07T03:50:51.826362+0000","last_clean":"2024-03-07T03:50:51.826362+0000","last_became_active":"2024-03-07T03:25:05.754809+0000","last_became_peered":"2024-03-07T03:25:05.754809+0000","last_unstale":"2024-03-07T03:50:51.826362+0000","last_undegraded":"2024-03-07T03:50:51.826362+0000","last_fullsized":"2024-03-07T03:50:51.826362+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:17:44.629246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1510,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828064+0000","last_change":"2024-03-07T03:24:36.577321+0000","last_active":"2024-03-07T03:50:51.828064+0000","last_peered":"2024-03-07T03:50:51.828064+0000","last_clean":"2024-03-07T03:50:51.828064+0000","last_became_active":"2024-03-07T03:24:36.576821+0000","last_became_peered":"2024-03-07T03:24:36.576821+0000","last_unstale":"2024-03-07T03:50:51.828064+0000","last_undegraded":"2024-03-07T03:50:51.828064+0000","last_fullsized":"2024-03-07T03:50:51.828064+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:22:15.850424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":1656,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712752+0000","last_change":"2024-03-07T03:25:05.760637+0000","last_active":"2024-03-07T03:50:52.712752+0000","last_peered":"2024-03-07T03:50:52.712752+0000","last_clean":"2024-03-07T03:50:52.712752+0000","last_became_active":"2024-03-07T03:25:05.759731+0000","last_became_peered":"2024-03-07T03:25:05.759731+0000","last_unstale":"2024-03-07T03:50:52.712752+0000","last_undegraded":"2024-03-07T03:50:52.712752+0000","last_fullsized":"2024-03-07T03:50:52.712752+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:37:00.229672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1505,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.820386+0000","last_change":"2024-03-07T03:25:05.758930+0000","last_active":"2024-03-07T03:50:51.820386+0000","last_peered":"2024-03-07T03:50:51.820386+0000","last_clean":"2024-03-07T03:50:51.820386+0000","last_became_active":"2024-03-07T03:25:05.752951+0000","last_became_peered":"2024-03-07T03:25:05.752951+0000","last_unstale":"2024-03-07T03:50:51.820386+0000","last_undegraded":"2024-03-07T03:50:51.820386+0000","last_fullsized":"2024-03-07T03:50:51.820386+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:57:46.946958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":1667,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.711863+0000","last_change":"2024-03-07T03:25:05.756613+0000","last_active":"2024-03-07T03:50:52.711863+0000","last_peered":"2024-03-07T03:50:52.711863+0000","last_clean":"2024-03-07T03:50:52.711863+0000","last_became_active":"2024-03-07T03:25:05.755569+0000","last_became_peered":"2024-03-07T03:25:05.755569+0000","last_unstale":"2024-03-07T03:50:52.711863+0000","last_undegraded":"2024-03-07T03:50:52.711863+0000","last_fullsized":"2024-03-07T03:50:52.711863+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:53:55.456725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1075,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.037505+0000","last_change":"2024-03-07T03:23:51.896947+0000","last_active":"2024-03-07T03:50:48.037505+0000","last_peered":"2024-03-07T03:50:48.037505+0000","last_clean":"2024-03-07T03:50:48.037505+0000","last_became_active":"2024-03-07T03:23:51.894905+0000","last_became_peered":"2024-03-07T03:23:51.894905+0000","last_unstale":"2024-03-07T03:50:48.037505+0000","last_undegraded":"2024-03-07T03:50:48.037505+0000","last_fullsized":"2024-03-07T03:50:48.037505+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:53:19.480650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1526,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136673+0000","last_change":"2024-03-07T03:24:36.563029+0000","last_active":"2024-03-07T03:50:52.136673+0000","last_peered":"2024-03-07T03:50:52.136673+0000","last_clean":"2024-03-07T03:50:52.136673+0000","last_became_active":"2024-03-07T03:24:36.551321+0000","last_became_peered":"2024-03-07T03:24:36.551321+0000","last_unstale":"2024-03-07T03:50:52.136673+0000","last_undegraded":"2024-03-07T03:50:52.136673+0000","last_fullsized":"2024-03-07T03:50:52.136673+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:24:02.449683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1638,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.725514+0000","last_change":"2024-03-07T03:25:05.745431+0000","last_active":"2024-03-07T03:50:52.725514+0000","last_peered":"2024-03-07T03:50:52.725514+0000","last_clean":"2024-03-07T03:50:52.725514+0000","last_became_active":"2024-03-07T03:25:05.743847+0000","last_became_peered":"2024-03-07T03:25:05.743847+0000","last_unstale":"2024-03-07T03:50:52.725514+0000","last_undegraded":"2024-03-07T03:50:52.725514+0000","last_fullsized":"2024-03-07T03:50:52.725514+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:49:40.151264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138676+0000","last_change":"2024-03-07T03:24:36.589494+0000","last_active":"2024-03-07T03:50:52.138676+0000","last_peered":"2024-03-07T03:50:52.138676+0000","last_clean":"2024-03-07T03:50:52.138676+0000","last_became_active":"2024-03-07T03:24:36.575936+0000","last_became_peered":"2024-03-07T03:24:36.575936+0000","last_unstale":"2024-03-07T03:50:52.138676+0000","last_undegraded":"2024-03-07T03:50:52.138676+0000","last_fullsized":"2024-03-07T03:50:52.138676+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:22:21.229952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1645,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712342+0000","last_change":"2024-03-07T03:21:07.226408+0000","last_active":"2024-03-07T03:50:52.712342+0000","last_peered":"2024-03-07T03:50:52.712342+0000","last_clean":"2024-03-07T03:50:52.712342+0000","last_became_active":"2024-03-07T03:21:07.222660+0000","last_became_peered":"2024-03-07T03:21:07.222660+0000","last_unstale":"2024-03-07T03:50:52.712342+0000","last_undegraded":"2024-03-07T03:50:52.712342+0000","last_fullsized":"2024-03-07T03:50:52.712342+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:57:31.735684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1636,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.719964+0000","last_change":"2024-03-07T03:23:51.867358+0000","last_active":"2024-03-07T03:50:52.719964+0000","last_peered":"2024-03-07T03:50:52.719964+0000","last_clean":"2024-03-07T03:50:52.719964+0000","last_became_active":"2024-03-07T03:23:51.866212+0000","last_became_peered":"2024-03-07T03:23:51.866212+0000","last_unstale":"2024-03-07T03:50:52.719964+0000","last_undegraded":"2024-03-07T03:50:52.719964+0000","last_fullsized":"2024-03-07T03:50:52.719964+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:10:54.727645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1091,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.717966+0000","last_change":"2024-03-07T03:25:05.742408+0000","last_active":"2024-03-07T03:50:52.717966+0000","last_peered":"2024-03-07T03:50:52.717966+0000","last_clean":"2024-03-07T03:50:52.717966+0000","last_became_active":"2024-03-07T03:25:05.742000+0000","last_became_peered":"2024-03-07T03:25:05.742000+0000","last_unstale":"2024-03-07T03:50:52.717966+0000","last_undegraded":"2024-03-07T03:50:52.717966+0000","last_fullsized":"2024-03-07T03:50:52.717966+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:33:59.846812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1663,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.711585+0000","last_change":"2024-03-07T03:23:51.890859+0000","last_active":"2024-03-07T03:50:52.711585+0000","last_peered":"2024-03-07T03:50:52.711585+0000","last_clean":"2024-03-07T03:50:52.711585+0000","last_became_active":"2024-03-07T03:23:51.889934+0000","last_became_peered":"2024-03-07T03:23:51.889934+0000","last_unstale":"2024-03-07T03:50:52.711585+0000","last_undegraded":"2024-03-07T03:50:52.711585+0000","last_fullsized":"2024-03-07T03:50:52.711585+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:48:28.789581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1663,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712302+0000","last_change":"2024-03-07T03:25:05.761992+0000","last_active":"2024-03-07T03:50:52.712302+0000","last_peered":"2024-03-07T03:50:52.712302+0000","last_clean":"2024-03-07T03:50:52.712302+0000","last_became_active":"2024-03-07T03:25:05.761137+0000","last_became_peered":"2024-03-07T03:25:05.761137+0000","last_unstale":"2024-03-07T03:50:52.712302+0000","last_undegraded":"2024-03-07T03:50:52.712302+0000","last_fullsized":"2024-03-07T03:50:52.712302+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:45:27.321398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1092,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712356+0000","last_change":"2024-03-07T03:21:38.181209+0000","last_active":"2024-03-07T03:50:52.712356+0000","last_peered":"2024-03-07T03:50:52.712356+0000","last_clean":"2024-03-07T03:50:52.712356+0000","last_became_active":"2024-03-07T03:21:38.180257+0000","last_became_peered":"2024-03-07T03:21:38.180257+0000","last_unstale":"2024-03-07T03:50:52.712356+0000","last_undegraded":"2024-03-07T03:50:52.712356+0000","last_fullsized":"2024-03-07T03:50:52.712356+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:04:43.623665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1070,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.722895+0000","last_change":"2024-03-07T03:25:05.745658+0000","last_active":"2024-03-07T03:50:52.722895+0000","last_peered":"2024-03-07T03:50:52.722895+0000","last_clean":"2024-03-07T03:50:52.722895+0000","last_became_active":"2024-03-07T03:25:05.745333+0000","last_became_peered":"2024-03-07T03:25:05.745333+0000","last_unstale":"2024-03-07T03:50:52.722895+0000","last_undegraded":"2024-03-07T03:50:52.722895+0000","last_fullsized":"2024-03-07T03:50:52.722895+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:50:08.866643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1509,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.135970+0000","last_change":"2024-03-07T03:24:36.587414+0000","last_active":"2024-03-07T03:50:52.135970+0000","last_peered":"2024-03-07T03:50:52.135970+0000","last_clean":"2024-03-07T03:50:52.135970+0000","last_became_active":"2024-03-07T03:24:36.580492+0000","last_became_peered":"2024-03-07T03:24:36.580492+0000","last_unstale":"2024-03-07T03:50:52.135970+0000","last_undegraded":"2024-03-07T03:50:52.135970+0000","last_fullsized":"2024-03-07T03:50:52.135970+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:14:27.280098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"288'161","reported_seq":1442,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712668+0000","last_change":"2024-03-07T03:21:07.213336+0000","last_active":"2024-03-07T03:50:52.712668+0000","last_peered":"2024-03-07T03:50:52.712668+0000","last_clean":"2024-03-07T03:50:52.712668+0000","last_became_active":"2024-03-07T03:21:07.212427+0000","last_became_peered":"2024-03-07T03:21:07.212427+0000","last_unstale":"2024-03-07T03:50:52.712668+0000","last_undegraded":"2024-03-07T03:50:52.712668+0000","last_fullsized":"2024-03-07T03:50:52.712668+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:03:44.169894+0000","last_clean_scrub_stamp":"2024-03-07T03:03:44.169894+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:00:42.812513+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":1070,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.826690+0000","last_change":"2024-03-07T03:23:51.895364+0000","last_active":"2024-03-07T03:50:51.826690+0000","last_peered":"2024-03-07T03:50:51.826690+0000","last_clean":"2024-03-07T03:50:51.826690+0000","last_became_active":"2024-03-07T03:23:51.894115+0000","last_became_peered":"2024-03-07T03:23:51.894115+0000","last_unstale":"2024-03-07T03:50:51.826690+0000","last_undegraded":"2024-03-07T03:50:51.826690+0000","last_fullsized":"2024-03-07T03:50:51.826690+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:39:22.323228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1644,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.718858+0000","last_change":"2024-03-07T03:21:38.217039+0000","last_active":"2024-03-07T03:50:52.718858+0000","last_peered":"2024-03-07T03:50:52.718858+0000","last_clean":"2024-03-07T03:50:52.718858+0000","last_became_active":"2024-03-07T03:21:38.208087+0000","last_became_peered":"2024-03-07T03:21:38.208087+0000","last_unstale":"2024-03-07T03:50:52.718858+0000","last_undegraded":"2024-03-07T03:50:52.718858+0000","last_fullsized":"2024-03-07T03:50:52.718858+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:27:20.515324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1638,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.720182+0000","last_change":"2024-03-07T03:23:51.866591+0000","last_active":"2024-03-07T03:50:52.720182+0000","last_peered":"2024-03-07T03:50:52.720182+0000","last_clean":"2024-03-07T03:50:52.720182+0000","last_became_active":"2024-03-07T03:23:51.866176+0000","last_became_peered":"2024-03-07T03:23:51.866176+0000","last_unstale":"2024-03-07T03:50:52.720182+0000","last_undegraded":"2024-03-07T03:50:52.720182+0000","last_fullsized":"2024-03-07T03:50:52.720182+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:17:18.757845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.825469+0000","last_change":"2024-03-07T03:24:36.575901+0000","last_active":"2024-03-07T03:50:51.825469+0000","last_peered":"2024-03-07T03:50:51.825469+0000","last_clean":"2024-03-07T03:50:51.825469+0000","last_became_active":"2024-03-07T03:24:36.575414+0000","last_became_peered":"2024-03-07T03:24:36.575414+0000","last_unstale":"2024-03-07T03:50:51.825469+0000","last_undegraded":"2024-03-07T03:50:51.825469+0000","last_fullsized":"2024-03-07T03:50:51.825469+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:40:41.261906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1075,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.825378+0000","last_change":"2024-03-07T03:24:36.565742+0000","last_active":"2024-03-07T03:50:51.825378+0000","last_peered":"2024-03-07T03:50:51.825378+0000","last_clean":"2024-03-07T03:50:51.825378+0000","last_became_active":"2024-03-07T03:24:36.565423+0000","last_became_peered":"2024-03-07T03:24:36.565423+0000","last_unstale":"2024-03-07T03:50:51.825378+0000","last_undegraded":"2024-03-07T03:50:51.825378+0000","last_fullsized":"2024-03-07T03:50:51.825378+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:32:37.673607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1523,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.822876+0000","last_change":"2024-03-07T03:25:05.750215+0000","last_active":"2024-03-07T03:50:51.822876+0000","last_peered":"2024-03-07T03:50:51.822876+0000","last_clean":"2024-03-07T03:50:51.822876+0000","last_became_active":"2024-03-07T03:25:05.749173+0000","last_became_peered":"2024-03-07T03:25:05.749173+0000","last_unstale":"2024-03-07T03:50:51.822876+0000","last_undegraded":"2024-03-07T03:50:51.822876+0000","last_fullsized":"2024-03-07T03:50:51.822876+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:51:20.791266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":1070,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823740+0000","last_change":"2024-03-07T03:21:38.190092+0000","last_active":"2024-03-07T03:50:51.823740+0000","last_peered":"2024-03-07T03:50:51.823740+0000","last_clean":"2024-03-07T03:50:51.823740+0000","last_became_active":"2024-03-07T03:21:38.189299+0000","last_became_peered":"2024-03-07T03:21:38.189299+0000","last_unstale":"2024-03-07T03:50:51.823740+0000","last_undegraded":"2024-03-07T03:50:51.823740+0000","last_fullsized":"2024-03-07T03:50:51.823740+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:32:45.193894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1637,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714919+0000","last_change":"2024-03-07T03:25:05.745377+0000","last_active":"2024-03-07T03:50:52.714919+0000","last_peered":"2024-03-07T03:50:52.714919+0000","last_clean":"2024-03-07T03:50:52.714919+0000","last_became_active":"2024-03-07T03:25:05.742616+0000","last_became_peered":"2024-03-07T03:25:05.742616+0000","last_unstale":"2024-03-07T03:50:52.714919+0000","last_undegraded":"2024-03-07T03:50:52.714919+0000","last_fullsized":"2024-03-07T03:50:52.714919+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:33:44.200407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1080,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828225+0000","last_change":"2024-03-07T03:24:36.552105+0000","last_active":"2024-03-07T03:50:51.828225+0000","last_peered":"2024-03-07T03:50:51.828225+0000","last_clean":"2024-03-07T03:50:51.828225+0000","last_became_active":"2024-03-07T03:24:36.551714+0000","last_became_peered":"2024-03-07T03:24:36.551714+0000","last_unstale":"2024-03-07T03:50:51.828225+0000","last_undegraded":"2024-03-07T03:50:51.828225+0000","last_fullsized":"2024-03-07T03:50:51.828225+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:05:39.898596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1649,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.720322+0000","last_change":"2024-03-07T03:21:38.217877+0000","last_active":"2024-03-07T03:50:52.720322+0000","last_peered":"2024-03-07T03:50:52.720322+0000","last_clean":"2024-03-07T03:50:52.720322+0000","last_became_active":"2024-03-07T03:21:38.217589+0000","last_became_peered":"2024-03-07T03:21:38.217589+0000","last_unstale":"2024-03-07T03:50:52.720322+0000","last_undegraded":"2024-03-07T03:50:52.720322+0000","last_fullsized":"2024-03-07T03:50:52.720322+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:09:30.305203+0000","last_clean_scrub_stamp":"2024-03-07T03:09:30.305203+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:56:05.365717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1645,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.718470+0000","last_change":"2024-03-07T03:25:05.745198+0000","last_active":"2024-03-07T03:50:52.718470+0000","last_peered":"2024-03-07T03:50:52.718470+0000","last_clean":"2024-03-07T03:50:52.718470+0000","last_became_active":"2024-03-07T03:25:05.744778+0000","last_became_peered":"2024-03-07T03:25:05.744778+0000","last_unstale":"2024-03-07T03:50:52.718470+0000","last_undegraded":"2024-03-07T03:50:52.718470+0000","last_fullsized":"2024-03-07T03:50:52.718470+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:38:00.822111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1515,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136409+0000","last_change":"2024-03-07T03:24:36.569612+0000","last_active":"2024-03-07T03:50:52.136409+0000","last_peered":"2024-03-07T03:50:52.136409+0000","last_clean":"2024-03-07T03:50:52.136409+0000","last_became_active":"2024-03-07T03:24:36.554169+0000","last_became_peered":"2024-03-07T03:24:36.554169+0000","last_unstale":"2024-03-07T03:50:52.136409+0000","last_undegraded":"2024-03-07T03:50:52.136409+0000","last_fullsized":"2024-03-07T03:50:52.136409+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:20:48.944499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1510,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.830096+0000","last_change":"2024-03-07T03:25:05.766616+0000","last_active":"2024-03-07T03:50:51.830096+0000","last_peered":"2024-03-07T03:50:51.830096+0000","last_clean":"2024-03-07T03:50:51.830096+0000","last_became_active":"2024-03-07T03:25:05.766291+0000","last_became_peered":"2024-03-07T03:25:05.766291+0000","last_unstale":"2024-03-07T03:50:51.830096+0000","last_undegraded":"2024-03-07T03:50:51.830096+0000","last_fullsized":"2024-03-07T03:50:51.830096+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T15:03:52.746085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1512,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.821130+0000","last_change":"2024-03-07T03:25:05.732068+0000","last_active":"2024-03-07T03:50:51.821130+0000","last_peered":"2024-03-07T03:50:51.821130+0000","last_clean":"2024-03-07T03:50:51.821130+0000","last_became_active":"2024-03-07T03:25:05.731620+0000","last_became_peered":"2024-03-07T03:25:05.731620+0000","last_unstale":"2024-03-07T03:50:51.821130+0000","last_undegraded":"2024-03-07T03:50:51.821130+0000","last_fullsized":"2024-03-07T03:50:51.821130+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:34:43.559628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1520,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823998+0000","last_change":"2024-03-07T03:25:05.751860+0000","last_active":"2024-03-07T03:50:51.823998+0000","last_peered":"2024-03-07T03:50:51.823998+0000","last_clean":"2024-03-07T03:50:51.823998+0000","last_became_active":"2024-03-07T03:25:05.751351+0000","last_became_peered":"2024-03-07T03:25:05.751351+0000","last_unstale":"2024-03-07T03:50:51.823998+0000","last_undegraded":"2024-03-07T03:50:51.823998+0000","last_fullsized":"2024-03-07T03:50:51.823998+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:17:13.168268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1085,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.039591+0000","last_change":"2024-03-07T03:25:05.740806+0000","last_active":"2024-03-07T03:50:48.039591+0000","last_peered":"2024-03-07T03:50:48.039591+0000","last_clean":"2024-03-07T03:50:48.039591+0000","last_became_active":"2024-03-07T03:25:05.740439+0000","last_became_peered":"2024-03-07T03:25:05.740439+0000","last_unstale":"2024-03-07T03:50:48.039591+0000","last_undegraded":"2024-03-07T03:50:48.039591+0000","last_fullsized":"2024-03-07T03:50:48.039591+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:29:36.069032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1527,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.827864+0000","last_change":"2024-03-07T03:25:05.764961+0000","last_active":"2024-03-07T03:50:51.827864+0000","last_peered":"2024-03-07T03:50:51.827864+0000","last_clean":"2024-03-07T03:50:51.827864+0000","last_became_active":"2024-03-07T03:25:05.764581+0000","last_became_peered":"2024-03-07T03:25:05.764581+0000","last_unstale":"2024-03-07T03:50:51.827864+0000","last_undegraded":"2024-03-07T03:50:51.827864+0000","last_fullsized":"2024-03-07T03:50:51.827864+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:44:56.467749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":1538,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136944+0000","last_change":"2024-03-07T03:25:05.741820+0000","last_active":"2024-03-07T03:50:52.136944+0000","last_peered":"2024-03-07T03:50:52.136944+0000","last_clean":"2024-03-07T03:50:52.136944+0000","last_became_active":"2024-03-07T03:25:05.741131+0000","last_became_peered":"2024-03-07T03:25:05.741131+0000","last_unstale":"2024-03-07T03:50:52.136944+0000","last_undegraded":"2024-03-07T03:50:52.136944+0000","last_fullsized":"2024-03-07T03:50:52.136944+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:53:27.540203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1662,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.711885+0000","last_change":"2024-03-07T03:25:05.760704+0000","last_active":"2024-03-07T03:50:52.711885+0000","last_peered":"2024-03-07T03:50:52.711885+0000","last_clean":"2024-03-07T03:50:52.711885+0000","last_became_active":"2024-03-07T03:25:05.759959+0000","last_became_peered":"2024-03-07T03:25:05.759959+0000","last_unstale":"2024-03-07T03:50:52.711885+0000","last_undegraded":"2024-03-07T03:50:52.711885+0000","last_fullsized":"2024-03-07T03:50:52.711885+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:14:32.135310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1064,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.825809+0000","last_change":"2024-03-07T03:24:36.562591+0000","last_active":"2024-03-07T03:50:51.825809+0000","last_peered":"2024-03-07T03:50:51.825809+0000","last_clean":"2024-03-07T03:50:51.825809+0000","last_became_active":"2024-03-07T03:24:36.561705+0000","last_became_peered":"2024-03-07T03:24:36.561705+0000","last_unstale":"2024-03-07T03:50:51.825809+0000","last_undegraded":"2024-03-07T03:50:51.825809+0000","last_fullsized":"2024-03-07T03:50:51.825809+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:10:00.333229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1078,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.037225+0000","last_change":"2024-03-07T03:23:51.872075+0000","last_active":"2024-03-07T03:50:48.037225+0000","last_peered":"2024-03-07T03:50:48.037225+0000","last_clean":"2024-03-07T03:50:48.037225+0000","last_became_active":"2024-03-07T03:23:51.869090+0000","last_became_peered":"2024-03-07T03:23:51.869090+0000","last_unstale":"2024-03-07T03:50:48.037225+0000","last_undegraded":"2024-03-07T03:50:48.037225+0000","last_fullsized":"2024-03-07T03:50:48.037225+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:56:43.404841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":1057,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.718105+0000","last_change":"2024-03-07T03:11:17.738764+0000","last_active":"2024-03-07T03:50:52.718105+0000","last_peered":"2024-03-07T03:50:52.718105+0000","last_clean":"2024-03-07T03:50:52.718105+0000","last_became_active":"2024-03-07T03:11:17.738151+0000","last_became_peered":"2024-03-07T03:11:17.738151+0000","last_unstale":"2024-03-07T03:50:52.718105+0000","last_undegraded":"2024-03-07T03:50:52.718105+0000","last_fullsized":"2024-03-07T03:50:52.718105+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:37:22.289110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1543,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.820673+0000","last_change":"2024-03-07T03:25:05.769855+0000","last_active":"2024-03-07T03:50:51.820673+0000","last_peered":"2024-03-07T03:50:51.820673+0000","last_clean":"2024-03-07T03:50:51.820673+0000","last_became_active":"2024-03-07T03:25:05.768412+0000","last_became_peered":"2024-03-07T03:25:05.768412+0000","last_unstale":"2024-03-07T03:50:51.820673+0000","last_undegraded":"2024-03-07T03:50:51.820673+0000","last_fullsized":"2024-03-07T03:50:51.820673+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:29:23.223181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":1089,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.713748+0000","last_change":"2024-03-07T03:25:05.740687+0000","last_active":"2024-03-07T03:50:52.713748+0000","last_peered":"2024-03-07T03:50:52.713748+0000","last_clean":"2024-03-07T03:50:52.713748+0000","last_became_active":"2024-03-07T03:25:05.738724+0000","last_became_peered":"2024-03-07T03:25:05.738724+0000","last_unstale":"2024-03-07T03:50:52.713748+0000","last_undegraded":"2024-03-07T03:50:52.713748+0000","last_fullsized":"2024-03-07T03:50:52.713748+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:13:31.913269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1048,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.037894+0000","last_change":"2024-03-07T03:11:17.846593+0000","last_active":"2024-03-07T03:50:48.037894+0000","last_peered":"2024-03-07T03:50:48.037894+0000","last_clean":"2024-03-07T03:50:48.037894+0000","last_became_active":"2024-03-07T03:11:17.845734+0000","last_became_peered":"2024-03-07T03:11:17.845734+0000","last_unstale":"2024-03-07T03:50:48.037894+0000","last_undegraded":"2024-03-07T03:50:48.037894+0000","last_fullsized":"2024-03-07T03:50:48.037894+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:18:40.161672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":1086,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.717325+0000","last_change":"2024-03-07T03:25:05.739761+0000","last_active":"2024-03-07T03:50:52.717325+0000","last_peered":"2024-03-07T03:50:52.717325+0000","last_clean":"2024-03-07T03:50:52.717325+0000","last_became_active":"2024-03-07T03:25:05.739351+0000","last_became_peered":"2024-03-07T03:25:05.739351+0000","last_unstale":"2024-03-07T03:50:52.717325+0000","last_undegraded":"2024-03-07T03:50:52.717325+0000","last_fullsized":"2024-03-07T03:50:52.717325+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:34:39.300885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1513,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138608+0000","last_change":"2024-03-07T03:24:36.575581+0000","last_active":"2024-03-07T03:50:52.138608+0000","last_peered":"2024-03-07T03:50:52.138608+0000","last_clean":"2024-03-07T03:50:52.138608+0000","last_became_active":"2024-03-07T03:24:36.562442+0000","last_became_peered":"2024-03-07T03:24:36.562442+0000","last_unstale":"2024-03-07T03:50:52.138608+0000","last_undegraded":"2024-03-07T03:50:52.138608+0000","last_fullsized":"2024-03-07T03:50:52.138608+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:27:06.200975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":1085,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712618+0000","last_change":"2024-03-07T03:21:07.224986+0000","last_active":"2024-03-07T03:50:52.712618+0000","last_peered":"2024-03-07T03:50:52.712618+0000","last_clean":"2024-03-07T03:50:52.712618+0000","last_became_active":"2024-03-07T03:21:07.223839+0000","last_became_peered":"2024-03-07T03:21:07.223839+0000","last_unstale":"2024-03-07T03:50:52.712618+0000","last_undegraded":"2024-03-07T03:50:52.712618+0000","last_fullsized":"2024-03-07T03:50:52.712618+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:29:44.461183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1651,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.709106+0000","last_change":"2024-03-07T03:24:36.586658+0000","last_active":"2024-03-07T03:50:52.709106+0000","last_peered":"2024-03-07T03:50:52.709106+0000","last_clean":"2024-03-07T03:50:52.709106+0000","last_became_active":"2024-03-07T03:24:36.586284+0000","last_became_peered":"2024-03-07T03:24:36.586284+0000","last_unstale":"2024-03-07T03:50:52.709106+0000","last_undegraded":"2024-03-07T03:50:52.709106+0000","last_fullsized":"2024-03-07T03:50:52.709106+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:16:06.802596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1069,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.712287+0000","last_change":"2024-03-07T03:25:05.749052+0000","last_active":"2024-03-07T03:50:52.712287+0000","last_peered":"2024-03-07T03:50:52.712287+0000","last_clean":"2024-03-07T03:50:52.712287+0000","last_became_active":"2024-03-07T03:25:05.742801+0000","last_became_peered":"2024-03-07T03:25:05.742801+0000","last_unstale":"2024-03-07T03:50:52.712287+0000","last_undegraded":"2024-03-07T03:50:52.712287+0000","last_fullsized":"2024-03-07T03:50:52.712287+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:15:09.074490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1061,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.040280+0000","last_change":"2024-03-07T03:24:36.546759+0000","last_active":"2024-03-07T03:50:48.040280+0000","last_peered":"2024-03-07T03:50:48.040280+0000","last_clean":"2024-03-07T03:50:48.040280+0000","last_became_active":"2024-03-07T03:24:36.545949+0000","last_became_peered":"2024-03-07T03:24:36.545949+0000","last_unstale":"2024-03-07T03:50:48.040280+0000","last_undegraded":"2024-03-07T03:50:48.040280+0000","last_fullsized":"2024-03-07T03:50:48.040280+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:33:39.293881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1652,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.710625+0000","last_change":"2024-03-07T03:24:36.566360+0000","last_active":"2024-03-07T03:50:52.710625+0000","last_peered":"2024-03-07T03:50:52.710625+0000","last_clean":"2024-03-07T03:50:52.710625+0000","last_became_active":"2024-03-07T03:24:36.565657+0000","last_became_peered":"2024-03-07T03:24:36.565657+0000","last_unstale":"2024-03-07T03:50:52.710625+0000","last_undegraded":"2024-03-07T03:50:52.710625+0000","last_fullsized":"2024-03-07T03:50:52.710625+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T12:44:47.754341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1513,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138073+0000","last_change":"2024-03-07T03:24:36.580300+0000","last_active":"2024-03-07T03:50:52.138073+0000","last_peered":"2024-03-07T03:50:52.138073+0000","last_clean":"2024-03-07T03:50:52.138073+0000","last_became_active":"2024-03-07T03:24:36.560593+0000","last_became_peered":"2024-03-07T03:24:36.560593+0000","last_unstale":"2024-03-07T03:50:52.138073+0000","last_undegraded":"2024-03-07T03:50:52.138073+0000","last_fullsized":"2024-03-07T03:50:52.138073+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T05:37:18.825686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":1088,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714554+0000","last_change":"2024-03-07T03:25:05.742630+0000","last_active":"2024-03-07T03:50:52.714554+0000","last_peered":"2024-03-07T03:50:52.714554+0000","last_clean":"2024-03-07T03:50:52.714554+0000","last_became_active":"2024-03-07T03:25:05.740480+0000","last_became_peered":"2024-03-07T03:25:05.740480+0000","last_unstale":"2024-03-07T03:50:52.714554+0000","last_undegraded":"2024-03-07T03:50:52.714554+0000","last_fullsized":"2024-03-07T03:50:52.714554+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:56:21.846159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1509,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.823033+0000","last_change":"2024-03-07T03:25:05.767010+0000","last_active":"2024-03-07T03:50:51.823033+0000","last_peered":"2024-03-07T03:50:51.823033+0000","last_clean":"2024-03-07T03:50:51.823033+0000","last_became_active":"2024-03-07T03:25:05.766702+0000","last_became_peered":"2024-03-07T03:25:05.766702+0000","last_unstale":"2024-03-07T03:50:51.823033+0000","last_undegraded":"2024-03-07T03:50:51.823033+0000","last_fullsized":"2024-03-07T03:50:51.823033+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T03:37:06.236762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"363'23","reported_seq":1542,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.828022+0000","last_change":"2024-03-07T03:25:05.735450+0000","last_active":"2024-03-07T03:50:51.828022+0000","last_peered":"2024-03-07T03:50:51.828022+0000","last_clean":"2024-03-07T03:50:51.828022+0000","last_became_active":"2024-03-07T03:25:05.734655+0000","last_became_peered":"2024-03-07T03:25:05.734655+0000","last_unstale":"2024-03-07T03:50:51.828022+0000","last_undegraded":"2024-03-07T03:50:51.828022+0000","last_fullsized":"2024-03-07T03:50:51.828022+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:34:59.285647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1636,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714535+0000","last_change":"2024-03-07T03:23:51.863430+0000","last_active":"2024-03-07T03:50:52.714535+0000","last_peered":"2024-03-07T03:50:52.714535+0000","last_clean":"2024-03-07T03:50:52.714535+0000","last_became_active":"2024-03-07T03:23:51.863022+0000","last_became_peered":"2024-03-07T03:23:51.863022+0000","last_unstale":"2024-03-07T03:50:52.714535+0000","last_undegraded":"2024-03-07T03:50:52.714535+0000","last_fullsized":"2024-03-07T03:50:52.714535+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T04:59:32.196016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":1089,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:47.859250+0000","last_change":"2024-03-07T03:25:05.740150+0000","last_active":"2024-03-07T03:50:47.859250+0000","last_peered":"2024-03-07T03:50:47.859250+0000","last_clean":"2024-03-07T03:50:47.859250+0000","last_became_active":"2024-03-07T03:25:05.738746+0000","last_became_peered":"2024-03-07T03:25:05.738746+0000","last_unstale":"2024-03-07T03:50:47.859250+0000","last_undegraded":"2024-03-07T03:50:47.859250+0000","last_fullsized":"2024-03-07T03:50:47.859250+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:25:03.546350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1506,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136287+0000","last_change":"2024-03-07T03:24:36.560614+0000","last_active":"2024-03-07T03:50:52.136287+0000","last_peered":"2024-03-07T03:50:52.136287+0000","last_clean":"2024-03-07T03:50:52.136287+0000","last_became_active":"2024-03-07T03:24:36.558250+0000","last_became_peered":"2024-03-07T03:24:36.558250+0000","last_unstale":"2024-03-07T03:50:52.136287+0000","last_undegraded":"2024-03-07T03:50:52.136287+0000","last_fullsized":"2024-03-07T03:50:52.136287+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T09:16:30.561539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1062,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.040199+0000","last_change":"2024-03-07T03:21:38.198598+0000","last_active":"2024-03-07T03:50:48.040199+0000","last_peered":"2024-03-07T03:50:48.040199+0000","last_clean":"2024-03-07T03:50:48.040199+0000","last_became_active":"2024-03-07T03:21:38.197849+0000","last_became_peered":"2024-03-07T03:21:38.197849+0000","last_unstale":"2024-03-07T03:50:48.040199+0000","last_undegraded":"2024-03-07T03:50:48.040199+0000","last_fullsized":"2024-03-07T03:50:48.040199+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T10:20:02.544918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1513,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136552+0000","last_change":"2024-03-07T03:24:36.521766+0000","last_active":"2024-03-07T03:50:52.136552+0000","last_peered":"2024-03-07T03:50:52.136552+0000","last_clean":"2024-03-07T03:50:52.136552+0000","last_became_active":"2024-03-07T03:24:36.520227+0000","last_became_peered":"2024-03-07T03:24:36.520227+0000","last_unstale":"2024-03-07T03:50:52.136552+0000","last_undegraded":"2024-03-07T03:50:52.136552+0000","last_fullsized":"2024-03-07T03:50:52.136552+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:30:26.946634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1651,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.710801+0000","last_change":"2024-03-07T03:24:36.553047+0000","last_active":"2024-03-07T03:50:52.710801+0000","last_peered":"2024-03-07T03:50:52.710801+0000","last_clean":"2024-03-07T03:50:52.710801+0000","last_became_active":"2024-03-07T03:24:36.552690+0000","last_became_peered":"2024-03-07T03:24:36.552690+0000","last_unstale":"2024-03-07T03:50:52.710801+0000","last_undegraded":"2024-03-07T03:50:52.710801+0000","last_fullsized":"2024-03-07T03:50:52.710801+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:34:00.975763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1070,"reported_epoch":1101,"state":"active+clean","last_fresh":"2024-03-07T03:50:48.236001+0000","last_change":"2024-03-07T03:21:38.200454+0000","last_active":"2024-03-07T03:50:48.236001+0000","last_peered":"2024-03-07T03:50:48.236001+0000","last_clean":"2024-03-07T03:50:48.236001+0000","last_became_active":"2024-03-07T03:21:38.199536+0000","last_became_peered":"2024-03-07T03:21:38.199536+0000","last_unstale":"2024-03-07T03:50:48.236001+0000","last_undegraded":"2024-03-07T03:50:48.236001+0000","last_fullsized":"2024-03-07T03:50:48.236001+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T13:59:25.711174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1529,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.136851+0000","last_change":"2024-03-07T03:24:36.560408+0000","last_active":"2024-03-07T03:50:52.136851+0000","last_peered":"2024-03-07T03:50:52.136851+0000","last_clean":"2024-03-07T03:50:52.136851+0000","last_became_active":"2024-03-07T03:24:36.548964+0000","last_became_peered":"2024-03-07T03:24:36.548964+0000","last_unstale":"2024-03-07T03:50:52.136851+0000","last_undegraded":"2024-03-07T03:50:52.136851+0000","last_fullsized":"2024-03-07T03:50:52.136851+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T07:40:11.900535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1655,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.709037+0000","last_change":"2024-03-07T03:24:36.587014+0000","last_active":"2024-03-07T03:50:52.709037+0000","last_peered":"2024-03-07T03:50:52.709037+0000","last_clean":"2024-03-07T03:50:52.709037+0000","last_became_active":"2024-03-07T03:24:36.586353+0000","last_became_peered":"2024-03-07T03:24:36.586353+0000","last_unstale":"2024-03-07T03:50:52.709037+0000","last_undegraded":"2024-03-07T03:50:52.709037+0000","last_fullsized":"2024-03-07T03:50:52.709037+0000","mapping_epoch":651,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":652,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T06:19:28.432486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1644,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.714054+0000","last_change":"2024-03-07T03:25:05.744751+0000","last_active":"2024-03-07T03:50:52.714054+0000","last_peered":"2024-03-07T03:50:52.714054+0000","last_clean":"2024-03-07T03:50:52.714054+0000","last_became_active":"2024-03-07T03:25:05.744404+0000","last_became_peered":"2024-03-07T03:25:05.744404+0000","last_unstale":"2024-03-07T03:50:52.714054+0000","last_undegraded":"2024-03-07T03:50:52.714054+0000","last_fullsized":"2024-03-07T03:50:52.714054+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T11:12:03.756012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1521,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:52.138153+0000","last_change":"2024-03-07T03:25:05.743998+0000","last_active":"2024-03-07T03:50:52.138153+0000","last_peered":"2024-03-07T03:50:52.138153+0000","last_clean":"2024-03-07T03:50:52.138153+0000","last_became_active":"2024-03-07T03:25:05.741440+0000","last_became_peered":"2024-03-07T03:25:05.741440+0000","last_unstale":"2024-03-07T03:50:52.138153+0000","last_undegraded":"2024-03-07T03:50:52.138153+0000","last_fullsized":"2024-03-07T03:50:52.138153+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:17.689136+0000","last_clean_scrub_stamp":"2024-03-07T03:11:17.689136+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T08:16:57.069373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":1527,"reported_epoch":1102,"state":"active+clean","last_fresh":"2024-03-07T03:50:51.824354+0000","last_change":"2024-03-07T03:25:05.767463+0000","last_active":"2024-03-07T03:50:51.824354+0000","last_peered":"2024-03-07T03:50:51.824354+0000","last_clean":"2024-03-07T03:50:51.824354+0000","last_became_active":"2024-03-07T03:25:05.766939+0000","last_became_peered":"2024-03-07T03:25:05.766939+0000","last_unstale":"2024-03-07T03:50:51.824354+0000","last_undegraded":"2024-03-07T03:50:51.824354+0000","last_fullsized":"2024-03-07T03:50:51.824354+0000","mapping_epoch":656,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":657,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-07T03:11:16.686114+0000","last_clean_scrub_stamp":"2024-03-07T03:11:16.686114+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-08T14:26:47.320990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":65,"num_read_kb":68,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3677,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201815,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112604,"kb_used_data":13460,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93632548,"statfs":{"total":95995035648,"available":95879729152,"internally_reserved":0,"allocated":13783040,"data_stored":8816519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":101513483},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463458,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":138836,"kb_used_data":13448,"kb_used_omap":1,"kb_used_meta":125374,"kb_avail":93606316,"statfs":{"total":95995035648,"available":95852867584,"internally_reserved":0,"allocated":13770752,"data_stored":8816373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":128383243},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":509,"seq":2186138354034,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76604,"kb_used_data":12852,"kb_used_omap":53,"kb_used_meta":63690,"kb_avail":93668548,"statfs":{"total":95995035648,"available":95916593152,"internally_reserved":0,"allocated":13160448,"data_stored":8221191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54606,"internal_metadata":65219250},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":478,"seq":2052994367864,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":85516,"kb_used_data":13440,"kb_used_omap":59,"kb_used_meta":72004,"kb_avail":93659636,"statfs":{"total":95995035648,"available":95907467264,"internally_reserved":0,"allocated":13762560,"data_stored":8812163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":60713,"internal_metadata":73732823},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":634,"seq":2723009266007,"num_pgs":18,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":71304,"kb_used_data":12860,"kb_used_omap":73,"kb_used_meta":58358,"kb_avail":93673848,"statfs":{"total":95995035648,"available":95922020352,"internally_reserved":0,"allocated":13168640,"data_stored":8221807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":75685,"internal_metadata":59758683},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":651,"seq":2796023710030,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":89348,"kb_used_data":12856,"kb_used_omap":67,"kb_used_meta":76412,"kb_avail":93655804,"statfs":{"total":95995035648,"available":95903543296,"internally_reserved":0,"allocated":13164544,"data_stored":8221281,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69425,"internal_metadata":78246095},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":656,"seq":2817498546504,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":62808,"kb_used_data":12876,"kb_used_omap":71,"kb_used_meta":49848,"kb_avail":93682344,"statfs":{"total":95995035648,"available":95930720256,"internally_reserved":0,"allocated":13185024,"data_stored":8226093,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72724,"internal_metadata":51045356},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725100,"num_pgs":22,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":84168,"kb_used_data":12860,"kb_used_omap":1,"kb_used_meta":71294,"kb_avail":93660984,"statfs":{"total":95995035648,"available":95908847616,"internally_reserved":0,"allocated":13168640,"data_stored":8221831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1781,"internal_metadata":73005323},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1504,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":899,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1274,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-07T03:52:19.670 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-07T03:52:19.670 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-07T03:52:19.670 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-07T03:52:19.670 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:20.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:19 smithi148 bash[79819]: audit 2024-03-07T03:52:18.855382+0000 mgr.y (mgr.34101) 1152 : audit 0 from='client.44963 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:52:20.051 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:19 smithi148 bash[82222]: audit 2024-03-07T03:52:18.855382+0000 mgr.y (mgr.34101) 1152 : audit 0 from='client.44963 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:52:20.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:19 smithi195 bash[66542]: audit 2024-03-07T03:52:18.855382+0000 mgr.y (mgr.34101) 1152 : audit 0 from='client.44963 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-07T03:52:21.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:20 smithi148 bash[79819]: cluster 2024-03-07T03:52:19.308478+0000 mgr.y (mgr.34101) 1153 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:21.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:20 smithi148 bash[82222]: cluster 2024-03-07T03:52:19.308478+0000 mgr.y (mgr.34101) 1153 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:21.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:20 smithi195 bash[66542]: cluster 2024-03-07T03:52:19.308478+0000 mgr.y (mgr.34101) 1153 : cluster 0 pgmap v1872: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:22.301 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:21 smithi148 bash[82222]: cluster 2024-03-07T03:52:21.309276+0000 mgr.y (mgr.34101) 1154 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:22.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:21 smithi148 bash[79819]: cluster 2024-03-07T03:52:21.309276+0000 mgr.y (mgr.34101) 1154 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:22.313 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:21 smithi195 bash[66542]: cluster 2024-03-07T03:52:21.309276+0000 mgr.y (mgr.34101) 1154 : cluster 0 pgmap v1873: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:22.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:52:22 smithi195 bash[59134]: ts=2024-03-07T03:52:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:24.409 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:24.801 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:24 smithi148 bash[82222]: cluster 2024-03-07T03:52:23.310099+0000 mgr.y (mgr.34101) 1155 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:24.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:24 smithi148 bash[79819]: cluster 2024-03-07T03:52:23.310099+0000 mgr.y (mgr.34101) 1155 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:24.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:24 smithi195 bash[66542]: cluster 2024-03-07T03:52:23.310099+0000 mgr.y (mgr.34101) 1155 : cluster 0 pgmap v1874: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:26.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[79819]: cluster 2024-03-07T03:52:25.311041+0000 mgr.y (mgr.34101) 1156 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:26.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[79819]: audit 2024-03-07T03:52:25.773564+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:52:26.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[79819]: audit 2024-03-07T03:52:26.300276+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:26.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[79819]: audit 2024-03-07T03:52:26.313297+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:26.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[82222]: cluster 2024-03-07T03:52:25.311041+0000 mgr.y (mgr.34101) 1156 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:26.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[82222]: audit 2024-03-07T03:52:25.773564+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:52:26.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[82222]: audit 2024-03-07T03:52:26.300276+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:26.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:26 smithi148 bash[82222]: audit 2024-03-07T03:52:26.313297+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:26.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:26 smithi195 bash[66542]: cluster 2024-03-07T03:52:25.311041+0000 mgr.y (mgr.34101) 1156 : cluster 0 pgmap v1875: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:26.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:26 smithi195 bash[66542]: audit 2024-03-07T03:52:25.773564+0000 mon.a (mon.0) 1984 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:52:26.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:26 smithi195 bash[66542]: audit 2024-03-07T03:52:26.300276+0000 mon.a (mon.0) 1985 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:26.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:26 smithi195 bash[66542]: audit 2024-03-07T03:52:26.313297+0000 mon.a (mon.0) 1986 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:27.301 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:52:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:52:27] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:52:27.383 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:52:27.383 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:27.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:27 smithi148 bash[79819]: audit 2024-03-07T03:52:27.356497+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:27.801 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:27 smithi148 bash[82222]: audit 2024-03-07T03:52:27.356497+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:27.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:27 smithi195 bash[66542]: audit 2024-03-07T03:52:27.356497+0000 mon.a (mon.0) 1987 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:28.801 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:28 smithi148 bash[82222]: cluster 2024-03-07T03:52:27.311763+0000 mgr.y (mgr.34101) 1157 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:28.801 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:28 smithi148 bash[82222]: audit 2024-03-07T03:52:27.385196+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.148:0/2909311605' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:28.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:28 smithi148 bash[79819]: cluster 2024-03-07T03:52:27.311763+0000 mgr.y (mgr.34101) 1157 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:28.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:28 smithi148 bash[79819]: audit 2024-03-07T03:52:27.385196+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.148:0/2909311605' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:28.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:28 smithi195 bash[66542]: cluster 2024-03-07T03:52:27.311763+0000 mgr.y (mgr.34101) 1157 : cluster 0 pgmap v1876: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:28.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:28 smithi195 bash[66542]: audit 2024-03-07T03:52:27.385196+0000 mon.c (mon.2) 87 : audit 0 from='client.? 172.21.15.148:0/2909311605' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:30.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:30 smithi148 bash[79819]: cluster 2024-03-07T03:52:29.312773+0000 mgr.y (mgr.34101) 1158 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:30.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:30 smithi148 bash[82222]: cluster 2024-03-07T03:52:29.312773+0000 mgr.y (mgr.34101) 1158 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:30.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:30 smithi195 bash[66542]: cluster 2024-03-07T03:52:29.312773+0000 mgr.y (mgr.34101) 1158 : cluster 0 pgmap v1877: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:31.126 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:32.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:52:32 smithi195 bash[59134]: ts=2024-03-07T03:52:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:33.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[79819]: cluster 2024-03-07T03:52:31.313503+0000 mgr.y (mgr.34101) 1159 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[79819]: audit 2024-03-07T03:52:31.618240+0000 mon.a (mon.0) 1988 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[79819]: audit 2024-03-07T03:52:31.625490+0000 mon.a (mon.0) 1989 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[79819]: audit 2024-03-07T03:52:32.264695+0000 mon.a (mon.0) 1990 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[79819]: audit 2024-03-07T03:52:32.275162+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[82222]: cluster 2024-03-07T03:52:31.313503+0000 mgr.y (mgr.34101) 1159 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[82222]: audit 2024-03-07T03:52:31.618240+0000 mon.a (mon.0) 1988 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[82222]: audit 2024-03-07T03:52:31.625490+0000 mon.a (mon.0) 1989 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[82222]: audit 2024-03-07T03:52:32.264695+0000 mon.a (mon.0) 1990 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:32 smithi148 bash[82222]: audit 2024-03-07T03:52:32.275162+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:32 smithi195 bash[66542]: cluster 2024-03-07T03:52:31.313503+0000 mgr.y (mgr.34101) 1159 : cluster 0 pgmap v1878: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:33.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:32 smithi195 bash[66542]: audit 2024-03-07T03:52:31.618240+0000 mon.a (mon.0) 1988 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:32 smithi195 bash[66542]: audit 2024-03-07T03:52:31.625490+0000 mon.a (mon.0) 1989 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:32 smithi195 bash[66542]: audit 2024-03-07T03:52:32.264695+0000 mon.a (mon.0) 1990 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:33.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:32 smithi195 bash[66542]: audit 2024-03-07T03:52:32.275162+0000 mon.a (mon.0) 1991 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:34.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:33 smithi148 bash[79819]: audit 2024-03-07T03:52:32.907154+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:52:34.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:33 smithi148 bash[79819]: audit 2024-03-07T03:52:32.908157+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:52:34.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:33 smithi148 bash[79819]: audit 2024-03-07T03:52:32.913695+0000 mon.a (mon.0) 1994 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:34.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:33 smithi148 bash[82222]: audit 2024-03-07T03:52:32.907154+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:52:34.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:33 smithi148 bash[82222]: audit 2024-03-07T03:52:32.908157+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:52:34.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:33 smithi148 bash[82222]: audit 2024-03-07T03:52:32.913695+0000 mon.a (mon.0) 1994 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:34.072 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:33 smithi195 bash[66542]: audit 2024-03-07T03:52:32.907154+0000 mon.a (mon.0) 1992 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:52:34.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:33 smithi195 bash[66542]: audit 2024-03-07T03:52:32.908157+0000 mon.a (mon.0) 1993 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:52:34.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:33 smithi195 bash[66542]: audit 2024-03-07T03:52:32.913695+0000 mon.a (mon.0) 1994 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:52:34.906 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:35.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:34 smithi148 bash[79819]: cluster 2024-03-07T03:52:33.314261+0000 mgr.y (mgr.34101) 1160 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:35.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:34 smithi148 bash[82222]: cluster 2024-03-07T03:52:33.314261+0000 mgr.y (mgr.34101) 1160 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:35.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:34 smithi195 bash[66542]: cluster 2024-03-07T03:52:33.314261+0000 mgr.y (mgr.34101) 1160 : cluster 0 pgmap v1879: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:36.172 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:35 smithi148 bash[79819]: cluster 2024-03-07T03:52:35.315178+0000 mgr.y (mgr.34101) 1161 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:36.172 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:35 smithi148 bash[82222]: cluster 2024-03-07T03:52:35.315178+0000 mgr.y (mgr.34101) 1161 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:36.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:35 smithi195 bash[66542]: cluster 2024-03-07T03:52:35.315178+0000 mgr.y (mgr.34101) 1161 : cluster 0 pgmap v1880: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:37.458 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:52:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:52:37] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:52:37.877 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:52:37.877 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:38.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:38 smithi148 bash[79819]: cluster 2024-03-07T03:52:37.315902+0000 mgr.y (mgr.34101) 1162 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:38.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:38 smithi148 bash[79819]: audit 2024-03-07T03:52:37.878953+0000 mon.a (mon.0) 1995 : audit 0 from='client.? 172.21.15.148:0/1764918384' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:38.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:38 smithi148 bash[82222]: cluster 2024-03-07T03:52:37.315902+0000 mgr.y (mgr.34101) 1162 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:38.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:38 smithi148 bash[82222]: audit 2024-03-07T03:52:37.878953+0000 mon.a (mon.0) 1995 : audit 0 from='client.? 172.21.15.148:0/1764918384' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:38.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:38 smithi195 bash[66542]: cluster 2024-03-07T03:52:37.315902+0000 mgr.y (mgr.34101) 1162 : cluster 0 pgmap v1881: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:38.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:38 smithi195 bash[66542]: audit 2024-03-07T03:52:37.878953+0000 mon.a (mon.0) 1995 : audit 0 from='client.? 172.21.15.148:0/1764918384' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:40.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:40 smithi148 bash[79819]: cluster 2024-03-07T03:52:39.316913+0000 mgr.y (mgr.34101) 1163 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:40.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:40 smithi148 bash[82222]: cluster 2024-03-07T03:52:39.316913+0000 mgr.y (mgr.34101) 1163 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:40.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:40 smithi195 bash[66542]: cluster 2024-03-07T03:52:39.316913+0000 mgr.y (mgr.34101) 1163 : cluster 0 pgmap v1882: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:41.550 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:42.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:42 smithi195 bash[66542]: cluster 2024-03-07T03:52:41.317768+0000 mgr.y (mgr.34101) 1164 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:42.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:42 smithi195 bash[66542]: audit 2024-03-07T03:52:42.357117+0000 mon.a (mon.0) 1996 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:42.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:52:42 smithi195 bash[59134]: ts=2024-03-07T03:52:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:42.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:42 smithi148 bash[79819]: cluster 2024-03-07T03:52:41.317768+0000 mgr.y (mgr.34101) 1164 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:42.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:42 smithi148 bash[79819]: audit 2024-03-07T03:52:42.357117+0000 mon.a (mon.0) 1996 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:42.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:42 smithi148 bash[82222]: cluster 2024-03-07T03:52:41.317768+0000 mgr.y (mgr.34101) 1164 : cluster 0 pgmap v1883: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:42.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:42 smithi148 bash[82222]: audit 2024-03-07T03:52:42.357117+0000 mon.a (mon.0) 1996 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:44.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:44 smithi148 bash[79819]: cluster 2024-03-07T03:52:43.318644+0000 mgr.y (mgr.34101) 1165 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:44.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:44 smithi148 bash[82222]: cluster 2024-03-07T03:52:43.318644+0000 mgr.y (mgr.34101) 1165 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:44.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:44 smithi195 bash[66542]: cluster 2024-03-07T03:52:43.318644+0000 mgr.y (mgr.34101) 1165 : cluster 0 pgmap v1884: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:46.284 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:46.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:46 smithi148 bash[79819]: cluster 2024-03-07T03:52:45.319547+0000 mgr.y (mgr.34101) 1166 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:46.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:46 smithi148 bash[82222]: cluster 2024-03-07T03:52:45.319547+0000 mgr.y (mgr.34101) 1166 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:46.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:46 smithi195 bash[66542]: cluster 2024-03-07T03:52:45.319547+0000 mgr.y (mgr.34101) 1166 : cluster 0 pgmap v1885: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:47.438 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:52:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:52:47] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:52:48.551 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:48 smithi148 bash[79819]: cluster 2024-03-07T03:52:47.320171+0000 mgr.y (mgr.34101) 1167 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:48.551 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:48 smithi148 bash[82222]: cluster 2024-03-07T03:52:47.320171+0000 mgr.y (mgr.34101) 1167 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:48.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:48 smithi195 bash[66542]: cluster 2024-03-07T03:52:47.320171+0000 mgr.y (mgr.34101) 1167 : cluster 0 pgmap v1886: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:49.377 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:52:49.377 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:52:49.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:49 smithi148 bash[79819]: audit 2024-03-07T03:52:49.378805+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.148:0/1230976055' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:49.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:49 smithi148 bash[82222]: audit 2024-03-07T03:52:49.378805+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.148:0/1230976055' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:49.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:49 smithi195 bash[66542]: audit 2024-03-07T03:52:49.378805+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.148:0/1230976055' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:52:50.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:50 smithi148 bash[79819]: cluster 2024-03-07T03:52:49.321151+0000 mgr.y (mgr.34101) 1168 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:50.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:50 smithi148 bash[82222]: cluster 2024-03-07T03:52:49.321151+0000 mgr.y (mgr.34101) 1168 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:50.822 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:50 smithi195 bash[66542]: cluster 2024-03-07T03:52:49.321151+0000 mgr.y (mgr.34101) 1168 : cluster 0 pgmap v1887: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:52.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:52 smithi195 bash[66542]: cluster 2024-03-07T03:52:51.321956+0000 mgr.y (mgr.34101) 1169 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:52.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:52:52 smithi195 bash[59134]: ts=2024-03-07T03:52:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:52:52.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:52 smithi148 bash[79819]: cluster 2024-03-07T03:52:51.321956+0000 mgr.y (mgr.34101) 1169 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:52.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:52 smithi148 bash[82222]: cluster 2024-03-07T03:52:51.321956+0000 mgr.y (mgr.34101) 1169 : cluster 0 pgmap v1888: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:53.084 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:52:55.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:54 smithi148 bash[79819]: cluster 2024-03-07T03:52:53.322865+0000 mgr.y (mgr.34101) 1170 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:55.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:54 smithi148 bash[82222]: cluster 2024-03-07T03:52:53.322865+0000 mgr.y (mgr.34101) 1170 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:55.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:54 smithi195 bash[66542]: cluster 2024-03-07T03:52:53.322865+0000 mgr.y (mgr.34101) 1170 : cluster 0 pgmap v1889: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.178 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:56 smithi148 bash[79819]: cluster 2024-03-07T03:52:55.323773+0000 mgr.y (mgr.34101) 1171 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.178 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:56 smithi148 bash[82222]: cluster 2024-03-07T03:52:55.323773+0000 mgr.y (mgr.34101) 1171 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:56 smithi195 bash[66542]: cluster 2024-03-07T03:52:55.323773+0000 mgr.y (mgr.34101) 1171 : cluster 0 pgmap v1890: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:57.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:52:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:52:57] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:52:57.821 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:52:58.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:57 smithi148 bash[79819]: audit 2024-03-07T03:52:57.357524+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:58.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:57 smithi148 bash[82222]: audit 2024-03-07T03:52:57.357524+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:58.322 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:57 smithi195 bash[66542]: audit 2024-03-07T03:52:57.357524+0000 mon.a (mon.0) 1997 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:52:59.102 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:52:58 smithi148 bash[79819]: cluster 2024-03-07T03:52:57.324394+0000 mgr.y (mgr.34101) 1172 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:59.102 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:52:58 smithi148 bash[82222]: cluster 2024-03-07T03:52:57.324394+0000 mgr.y (mgr.34101) 1172 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:52:59.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:52:58 smithi195 bash[66542]: cluster 2024-03-07T03:52:57.324394+0000 mgr.y (mgr.34101) 1172 : cluster 0 pgmap v1891: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:00.753 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:53:00.754 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:01.051 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:00 smithi148 bash[79819]: cluster 2024-03-07T03:52:59.325346+0000 mgr.y (mgr.34101) 1173 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:01.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:00 smithi148 bash[79819]: audit 2024-03-07T03:53:00.755590+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.148:0/1877245974' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:01.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:00 smithi148 bash[82222]: cluster 2024-03-07T03:52:59.325346+0000 mgr.y (mgr.34101) 1173 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:01.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:00 smithi148 bash[82222]: audit 2024-03-07T03:53:00.755590+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.148:0/1877245974' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:01.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:00 smithi195 bash[66542]: cluster 2024-03-07T03:52:59.325346+0000 mgr.y (mgr.34101) 1173 : cluster 0 pgmap v1892: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:01.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:00 smithi195 bash[66542]: audit 2024-03-07T03:53:00.755590+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.148:0/1877245974' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:02.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:53:02 smithi195 bash[59134]: ts=2024-03-07T03:53:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:03.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:02 smithi148 bash[79819]: cluster 2024-03-07T03:53:01.326176+0000 mgr.y (mgr.34101) 1174 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:02 smithi148 bash[82222]: cluster 2024-03-07T03:53:01.326176+0000 mgr.y (mgr.34101) 1174 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:03.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:02 smithi195 bash[66542]: cluster 2024-03-07T03:53:01.326176+0000 mgr.y (mgr.34101) 1174 : cluster 0 pgmap v1893: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:04.471 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:05.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:04 smithi148 bash[79819]: cluster 2024-03-07T03:53:03.327040+0000 mgr.y (mgr.34101) 1175 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:05.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:04 smithi148 bash[82222]: cluster 2024-03-07T03:53:03.327040+0000 mgr.y (mgr.34101) 1175 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:05.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:04 smithi195 bash[66542]: cluster 2024-03-07T03:53:03.327040+0000 mgr.y (mgr.34101) 1175 : cluster 0 pgmap v1894: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:07.176 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:06 smithi148 bash[79819]: cluster 2024-03-07T03:53:05.327974+0000 mgr.y (mgr.34101) 1176 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:07.176 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:06 smithi148 bash[82222]: cluster 2024-03-07T03:53:05.327974+0000 mgr.y (mgr.34101) 1176 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:07.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:06 smithi195 bash[66542]: cluster 2024-03-07T03:53:05.327974+0000 mgr.y (mgr.34101) 1176 : cluster 0 pgmap v1895: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:07.551 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:53:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:53:07] "GET /metrics HTTP/1.1" 200 34307 "" "Prometheus/2.43.0" 2024-03-07T03:53:09.209 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:09.232 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:08 smithi148 bash[79819]: cluster 2024-03-07T03:53:07.328743+0000 mgr.y (mgr.34101) 1177 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:09.232 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:08 smithi148 bash[82222]: cluster 2024-03-07T03:53:07.328743+0000 mgr.y (mgr.34101) 1177 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:09.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:08 smithi195 bash[66542]: cluster 2024-03-07T03:53:07.328743+0000 mgr.y (mgr.34101) 1177 : cluster 0 pgmap v1896: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:11.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:10 smithi148 bash[79819]: cluster 2024-03-07T03:53:09.329705+0000 mgr.y (mgr.34101) 1178 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:11.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:10 smithi148 bash[82222]: cluster 2024-03-07T03:53:09.329705+0000 mgr.y (mgr.34101) 1178 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:11.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:10 smithi195 bash[66542]: cluster 2024-03-07T03:53:09.329705+0000 mgr.y (mgr.34101) 1178 : cluster 0 pgmap v1897: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:12.172 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:53:12.172 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:12.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:53:12 smithi195 bash[59134]: ts=2024-03-07T03:53:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:13.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:12 smithi148 bash[79819]: cluster 2024-03-07T03:53:11.330540+0000 mgr.y (mgr.34101) 1179 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:13.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:12 smithi148 bash[79819]: audit 2024-03-07T03:53:12.173758+0000 mon.a (mon.0) 1998 : audit 0 from='client.? 172.21.15.148:0/29591875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:13.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:12 smithi148 bash[79819]: audit 2024-03-07T03:53:12.358084+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:13.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:12 smithi148 bash[82222]: cluster 2024-03-07T03:53:11.330540+0000 mgr.y (mgr.34101) 1179 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:13.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:12 smithi148 bash[82222]: audit 2024-03-07T03:53:12.173758+0000 mon.a (mon.0) 1998 : audit 0 from='client.? 172.21.15.148:0/29591875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:13.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:12 smithi148 bash[82222]: audit 2024-03-07T03:53:12.358084+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:13.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:12 smithi195 bash[66542]: cluster 2024-03-07T03:53:11.330540+0000 mgr.y (mgr.34101) 1179 : cluster 0 pgmap v1898: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:13.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:12 smithi195 bash[66542]: audit 2024-03-07T03:53:12.173758+0000 mon.a (mon.0) 1998 : audit 0 from='client.? 172.21.15.148:0/29591875' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:13.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:12 smithi195 bash[66542]: audit 2024-03-07T03:53:12.358084+0000 mon.a (mon.0) 1999 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:15.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:14 smithi148 bash[79819]: cluster 2024-03-07T03:53:13.331467+0000 mgr.y (mgr.34101) 1180 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:15.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:14 smithi148 bash[82222]: cluster 2024-03-07T03:53:13.331467+0000 mgr.y (mgr.34101) 1180 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:15.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:14 smithi195 bash[66542]: cluster 2024-03-07T03:53:13.331467+0000 mgr.y (mgr.34101) 1180 : cluster 0 pgmap v1899: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:15.815 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:17.178 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:16 smithi148 bash[79819]: cluster 2024-03-07T03:53:15.332553+0000 mgr.y (mgr.34101) 1181 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:17.178 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:16 smithi148 bash[82222]: cluster 2024-03-07T03:53:15.332553+0000 mgr.y (mgr.34101) 1181 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:17.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:16 smithi195 bash[66542]: cluster 2024-03-07T03:53:15.332553+0000 mgr.y (mgr.34101) 1181 : cluster 0 pgmap v1900: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:17.551 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:53:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:53:17] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:53:19.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:18 smithi148 bash[79819]: cluster 2024-03-07T03:53:17.333413+0000 mgr.y (mgr.34101) 1182 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:19.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:18 smithi148 bash[82222]: cluster 2024-03-07T03:53:17.333413+0000 mgr.y (mgr.34101) 1182 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:19.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:18 smithi195 bash[66542]: cluster 2024-03-07T03:53:17.333413+0000 mgr.y (mgr.34101) 1182 : cluster 0 pgmap v1901: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:20.566 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:21.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:20 smithi148 bash[79819]: cluster 2024-03-07T03:53:19.334456+0000 mgr.y (mgr.34101) 1183 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:21.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:20 smithi148 bash[82222]: cluster 2024-03-07T03:53:19.334456+0000 mgr.y (mgr.34101) 1183 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:21.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:20 smithi195 bash[66542]: cluster 2024-03-07T03:53:19.334456+0000 mgr.y (mgr.34101) 1183 : cluster 0 pgmap v1902: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:22.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:53:22 smithi195 bash[59134]: ts=2024-03-07T03:53:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:23.301 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:22 smithi148 bash[79819]: cluster 2024-03-07T03:53:21.335270+0000 mgr.y (mgr.34101) 1184 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:23.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:22 smithi148 bash[82222]: cluster 2024-03-07T03:53:21.335270+0000 mgr.y (mgr.34101) 1184 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:23.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:22 smithi195 bash[66542]: cluster 2024-03-07T03:53:21.335270+0000 mgr.y (mgr.34101) 1184 : cluster 0 pgmap v1903: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:23.526 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:53:23.527 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:24.280 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:23 smithi148 bash[79819]: audit 2024-03-07T03:53:23.528356+0000 mon.a (mon.0) 2000 : audit 0 from='client.? 172.21.15.148:0/1334342445' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:24.280 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:23 smithi148 bash[82222]: audit 2024-03-07T03:53:23.528356+0000 mon.a (mon.0) 2000 : audit 0 from='client.? 172.21.15.148:0/1334342445' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:24.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:23 smithi195 bash[66542]: audit 2024-03-07T03:53:23.528356+0000 mon.a (mon.0) 2000 : audit 0 from='client.? 172.21.15.148:0/1334342445' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:25.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:24 smithi148 bash[79819]: cluster 2024-03-07T03:53:23.335910+0000 mgr.y (mgr.34101) 1185 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:25.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:24 smithi148 bash[82222]: cluster 2024-03-07T03:53:23.335910+0000 mgr.y (mgr.34101) 1185 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:25.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:24 smithi195 bash[66542]: cluster 2024-03-07T03:53:23.335910+0000 mgr.y (mgr.34101) 1185 : cluster 0 pgmap v1904: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:27.178 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:26 smithi148 bash[79819]: cluster 2024-03-07T03:53:25.336827+0000 mgr.y (mgr.34101) 1186 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:27.178 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:26 smithi148 bash[82222]: cluster 2024-03-07T03:53:25.336827+0000 mgr.y (mgr.34101) 1186 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:27.282 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:27.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:26 smithi195 bash[66542]: cluster 2024-03-07T03:53:25.336827+0000 mgr.y (mgr.34101) 1186 : cluster 0 pgmap v1905: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:27.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:53:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:53:27] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:53:28.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:27 smithi148 bash[79819]: audit 2024-03-07T03:53:27.358521+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:28.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:27 smithi148 bash[82222]: audit 2024-03-07T03:53:27.358521+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:28.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:27 smithi195 bash[66542]: audit 2024-03-07T03:53:27.358521+0000 mon.a (mon.0) 2001 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:29.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:28 smithi148 bash[79819]: cluster 2024-03-07T03:53:27.337547+0000 mgr.y (mgr.34101) 1187 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:29.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:28 smithi148 bash[82222]: cluster 2024-03-07T03:53:27.337547+0000 mgr.y (mgr.34101) 1187 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:29.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:28 smithi195 bash[66542]: cluster 2024-03-07T03:53:27.337547+0000 mgr.y (mgr.34101) 1187 : cluster 0 pgmap v1906: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:30.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:29 smithi148 bash[79819]: cluster 2024-03-07T03:53:29.338632+0000 mgr.y (mgr.34101) 1188 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:30.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:29 smithi148 bash[82222]: cluster 2024-03-07T03:53:29.338632+0000 mgr.y (mgr.34101) 1188 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:30.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:29 smithi195 bash[66542]: cluster 2024-03-07T03:53:29.338632+0000 mgr.y (mgr.34101) 1188 : cluster 0 pgmap v1907: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:32.018 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:32.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:32 smithi195 bash[66542]: cluster 2024-03-07T03:53:31.339495+0000 mgr.y (mgr.34101) 1189 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:32.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:53:32 smithi195 bash[59134]: ts=2024-03-07T03:53:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:32.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:32 smithi148 bash[79819]: cluster 2024-03-07T03:53:31.339495+0000 mgr.y (mgr.34101) 1189 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:32.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:32 smithi148 bash[82222]: cluster 2024-03-07T03:53:31.339495+0000 mgr.y (mgr.34101) 1189 : cluster 0 pgmap v1908: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:33.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:33 smithi148 bash[79819]: audit 2024-03-07T03:53:32.956215+0000 mon.a (mon.0) 2002 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:53:33.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:33 smithi148 bash[82222]: audit 2024-03-07T03:53:32.956215+0000 mon.a (mon.0) 2002 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:53:33.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:33 smithi195 bash[66542]: audit 2024-03-07T03:53:32.956215+0000 mon.a (mon.0) 2002 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:53:34.743 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:34 smithi148 bash[79819]: cluster 2024-03-07T03:53:33.340388+0000 mgr.y (mgr.34101) 1190 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:34.743 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:34 smithi148 bash[82222]: cluster 2024-03-07T03:53:33.340388+0000 mgr.y (mgr.34101) 1190 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:34.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:34 smithi195 bash[66542]: cluster 2024-03-07T03:53:33.340388+0000 mgr.y (mgr.34101) 1190 : cluster 0 pgmap v1909: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:35.184 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:53:35.184 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:35.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:35 smithi148 bash[79819]: audit 2024-03-07T03:53:35.185863+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.148:0/785348267' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:35.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:35 smithi148 bash[82222]: audit 2024-03-07T03:53:35.185863+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.148:0/785348267' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:35.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:35 smithi195 bash[66542]: audit 2024-03-07T03:53:35.185863+0000 mon.c (mon.2) 90 : audit 0 from='client.? 172.21.15.148:0/785348267' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:36.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:36 smithi148 bash[79819]: cluster 2024-03-07T03:53:35.340983+0000 mgr.y (mgr.34101) 1191 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:36.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:36 smithi148 bash[82222]: cluster 2024-03-07T03:53:35.340983+0000 mgr.y (mgr.34101) 1191 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:36.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:36 smithi195 bash[66542]: cluster 2024-03-07T03:53:35.340983+0000 mgr.y (mgr.34101) 1191 : cluster 0 pgmap v1910: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:37.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:53:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:53:37] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:53:38.801 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:38 smithi148 bash[79819]: cluster 2024-03-07T03:53:37.341664+0000 mgr.y (mgr.34101) 1192 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:38.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:38 smithi148 bash[82222]: cluster 2024-03-07T03:53:37.341664+0000 mgr.y (mgr.34101) 1192 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:38.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:38 smithi195 bash[66542]: cluster 2024-03-07T03:53:37.341664+0000 mgr.y (mgr.34101) 1192 : cluster 0 pgmap v1911: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:38.866 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:40.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: audit 2024-03-07T03:53:39.044217+0000 mon.a (mon.0) 2003 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: audit 2024-03-07T03:53:39.049222+0000 mon.a (mon.0) 2004 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: audit 2024-03-07T03:53:39.151734+0000 mon.a (mon.0) 2005 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: audit 2024-03-07T03:53:39.160564+0000 mon.a (mon.0) 2006 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: cluster 2024-03-07T03:53:39.342640+0000 mgr.y (mgr.34101) 1193 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: audit 2024-03-07T03:53:39.740973+0000 mon.a (mon.0) 2007 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: audit 2024-03-07T03:53:39.742516+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[79819]: audit 2024-03-07T03:53:39.749849+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: audit 2024-03-07T03:53:39.044217+0000 mon.a (mon.0) 2003 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: audit 2024-03-07T03:53:39.049222+0000 mon.a (mon.0) 2004 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: audit 2024-03-07T03:53:39.151734+0000 mon.a (mon.0) 2005 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: audit 2024-03-07T03:53:39.160564+0000 mon.a (mon.0) 2006 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: cluster 2024-03-07T03:53:39.342640+0000 mgr.y (mgr.34101) 1193 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:40.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: audit 2024-03-07T03:53:39.740973+0000 mon.a (mon.0) 2007 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:53:40.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: audit 2024-03-07T03:53:39.742516+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:53:40.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:40 smithi148 bash[82222]: audit 2024-03-07T03:53:39.749849+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: audit 2024-03-07T03:53:39.044217+0000 mon.a (mon.0) 2003 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: audit 2024-03-07T03:53:39.049222+0000 mon.a (mon.0) 2004 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: audit 2024-03-07T03:53:39.151734+0000 mon.a (mon.0) 2005 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: audit 2024-03-07T03:53:39.160564+0000 mon.a (mon.0) 2006 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:40.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: cluster 2024-03-07T03:53:39.342640+0000 mgr.y (mgr.34101) 1193 : cluster 0 pgmap v1912: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:40.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: audit 2024-03-07T03:53:39.740973+0000 mon.a (mon.0) 2007 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:53:40.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: audit 2024-03-07T03:53:39.742516+0000 mon.a (mon.0) 2008 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:53:40.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:40 smithi195 bash[66542]: audit 2024-03-07T03:53:39.749849+0000 mon.a (mon.0) 2009 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:53:42.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:53:42 smithi195 bash[59134]: ts=2024-03-07T03:53:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:42 smithi195 bash[66542]: cluster 2024-03-07T03:53:41.343474+0000 mgr.y (mgr.34101) 1194 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:42 smithi195 bash[66542]: audit 2024-03-07T03:53:42.359041+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:42.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:42 smithi148 bash[79819]: cluster 2024-03-07T03:53:41.343474+0000 mgr.y (mgr.34101) 1194 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:42.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:42 smithi148 bash[79819]: audit 2024-03-07T03:53:42.359041+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:42.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:42 smithi148 bash[82222]: cluster 2024-03-07T03:53:41.343474+0000 mgr.y (mgr.34101) 1194 : cluster 0 pgmap v1913: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:42.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:42 smithi148 bash[82222]: audit 2024-03-07T03:53:42.359041+0000 mon.a (mon.0) 2010 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:43.603 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:44.667 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:44 smithi148 bash[79819]: cluster 2024-03-07T03:53:43.344321+0000 mgr.y (mgr.34101) 1195 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:44.667 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:44 smithi148 bash[82222]: cluster 2024-03-07T03:53:43.344321+0000 mgr.y (mgr.34101) 1195 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:44.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:44 smithi195 bash[66542]: cluster 2024-03-07T03:53:43.344321+0000 mgr.y (mgr.34101) 1195 : cluster 0 pgmap v1914: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:46.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:46 smithi148 bash[79819]: cluster 2024-03-07T03:53:45.345184+0000 mgr.y (mgr.34101) 1196 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:46.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:46 smithi148 bash[82222]: cluster 2024-03-07T03:53:45.345184+0000 mgr.y (mgr.34101) 1196 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:46.668 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:53:46.668 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:46.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:46 smithi195 bash[66542]: cluster 2024-03-07T03:53:45.345184+0000 mgr.y (mgr.34101) 1196 : cluster 0 pgmap v1915: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:47.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:47 smithi148 bash[79819]: audit 2024-03-07T03:53:46.669964+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.148:0/782947973' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:47.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:53:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:53:47] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.43.0" 2024-03-07T03:53:47.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:47 smithi148 bash[82222]: audit 2024-03-07T03:53:46.669964+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.148:0/782947973' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:47.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:47 smithi195 bash[66542]: audit 2024-03-07T03:53:46.669964+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.148:0/782947973' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:49.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:48 smithi148 bash[79819]: cluster 2024-03-07T03:53:47.346051+0000 mgr.y (mgr.34101) 1197 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:49.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:48 smithi148 bash[82222]: cluster 2024-03-07T03:53:47.346051+0000 mgr.y (mgr.34101) 1197 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:49.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:48 smithi195 bash[66542]: cluster 2024-03-07T03:53:47.346051+0000 mgr.y (mgr.34101) 1197 : cluster 0 pgmap v1916: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:50.426 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:53:51.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:50 smithi148 bash[82222]: cluster 2024-03-07T03:53:49.347060+0000 mgr.y (mgr.34101) 1198 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:51.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:50 smithi148 bash[79819]: cluster 2024-03-07T03:53:49.347060+0000 mgr.y (mgr.34101) 1198 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:51.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:50 smithi195 bash[66542]: cluster 2024-03-07T03:53:49.347060+0000 mgr.y (mgr.34101) 1198 : cluster 0 pgmap v1917: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:52.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:53:52 smithi195 bash[59134]: ts=2024-03-07T03:53:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:53:53.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:52 smithi148 bash[79819]: cluster 2024-03-07T03:53:51.347871+0000 mgr.y (mgr.34101) 1199 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:53.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:52 smithi148 bash[82222]: cluster 2024-03-07T03:53:51.347871+0000 mgr.y (mgr.34101) 1199 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:53.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:52 smithi195 bash[66542]: cluster 2024-03-07T03:53:51.347871+0000 mgr.y (mgr.34101) 1199 : cluster 0 pgmap v1918: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:55.163 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:53:55.185 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:54 smithi148 bash[79819]: cluster 2024-03-07T03:53:53.348738+0000 mgr.y (mgr.34101) 1200 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:55.185 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:54 smithi148 bash[82222]: cluster 2024-03-07T03:53:53.348738+0000 mgr.y (mgr.34101) 1200 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:55.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:54 smithi195 bash[66542]: cluster 2024-03-07T03:53:53.348738+0000 mgr.y (mgr.34101) 1200 : cluster 0 pgmap v1919: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:57.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:56 smithi148 bash[82222]: cluster 2024-03-07T03:53:55.349607+0000 mgr.y (mgr.34101) 1201 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:57.178 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:56 smithi148 bash[79819]: cluster 2024-03-07T03:53:55.349607+0000 mgr.y (mgr.34101) 1201 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:57.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:56 smithi195 bash[66542]: cluster 2024-03-07T03:53:55.349607+0000 mgr.y (mgr.34101) 1201 : cluster 0 pgmap v1920: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:57.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:53:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:53:57] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.43.0" 2024-03-07T03:53:58.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:57 smithi148 bash[79819]: audit 2024-03-07T03:53:57.359439+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:58.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:57 smithi148 bash[82222]: audit 2024-03-07T03:53:57.359439+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:58.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:57 smithi195 bash[66542]: audit 2024-03-07T03:53:57.359439+0000 mon.a (mon.0) 2011 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:53:58.359 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:53:58.359 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:53:59.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:58 smithi148 bash[79819]: cluster 2024-03-07T03:53:57.350447+0000 mgr.y (mgr.34101) 1202 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:59.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:53:58 smithi148 bash[79819]: audit 2024-03-07T03:53:58.360967+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.148:0/3872903248' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:59.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:58 smithi148 bash[82222]: cluster 2024-03-07T03:53:57.350447+0000 mgr.y (mgr.34101) 1202 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:59.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:53:58 smithi148 bash[82222]: audit 2024-03-07T03:53:58.360967+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.148:0/3872903248' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:53:59.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:58 smithi195 bash[66542]: cluster 2024-03-07T03:53:57.350447+0000 mgr.y (mgr.34101) 1202 : cluster 0 pgmap v1921: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:53:59.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:53:58 smithi195 bash[66542]: audit 2024-03-07T03:53:58.360967+0000 mon.c (mon.2) 92 : audit 0 from='client.? 172.21.15.148:0/3872903248' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:01.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:00 smithi148 bash[79819]: cluster 2024-03-07T03:53:59.351465+0000 mgr.y (mgr.34101) 1203 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:01.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:00 smithi148 bash[82222]: cluster 2024-03-07T03:53:59.351465+0000 mgr.y (mgr.34101) 1203 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:01.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:00 smithi195 bash[66542]: cluster 2024-03-07T03:53:59.351465+0000 mgr.y (mgr.34101) 1203 : cluster 0 pgmap v1922: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:02.092 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:02.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:54:02 smithi195 bash[59134]: ts=2024-03-07T03:54:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:03.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:02 smithi148 bash[79819]: cluster 2024-03-07T03:54:01.352353+0000 mgr.y (mgr.34101) 1204 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:03.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:02 smithi148 bash[82222]: cluster 2024-03-07T03:54:01.352353+0000 mgr.y (mgr.34101) 1204 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:03.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:02 smithi195 bash[66542]: cluster 2024-03-07T03:54:01.352353+0000 mgr.y (mgr.34101) 1204 : cluster 0 pgmap v1923: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:05.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:04 smithi148 bash[79819]: cluster 2024-03-07T03:54:03.353201+0000 mgr.y (mgr.34101) 1205 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:05.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:04 smithi148 bash[82222]: cluster 2024-03-07T03:54:03.353201+0000 mgr.y (mgr.34101) 1205 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:05.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:04 smithi195 bash[66542]: cluster 2024-03-07T03:54:03.353201+0000 mgr.y (mgr.34101) 1205 : cluster 0 pgmap v1924: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:06.829 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:07.177 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:06 smithi148 bash[79819]: cluster 2024-03-07T03:54:05.354176+0000 mgr.y (mgr.34101) 1206 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:07.178 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:06 smithi148 bash[82222]: cluster 2024-03-07T03:54:05.354176+0000 mgr.y (mgr.34101) 1206 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:07.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:06 smithi195 bash[66542]: cluster 2024-03-07T03:54:05.354176+0000 mgr.y (mgr.34101) 1206 : cluster 0 pgmap v1925: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:07.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:54:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:54:07] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.43.0" 2024-03-07T03:54:09.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:08 smithi148 bash[79819]: cluster 2024-03-07T03:54:07.354867+0000 mgr.y (mgr.34101) 1207 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:09.052 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:08 smithi148 bash[82222]: cluster 2024-03-07T03:54:07.354867+0000 mgr.y (mgr.34101) 1207 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:09.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:08 smithi195 bash[66542]: cluster 2024-03-07T03:54:07.354867+0000 mgr.y (mgr.34101) 1207 : cluster 0 pgmap v1926: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:09.895 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:54:09.896 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:10.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:09 smithi148 bash[79819]: audit 2024-03-07T03:54:09.897055+0000 mon.a (mon.0) 2012 : audit 0 from='client.? 172.21.15.148:0/4068973079' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:10.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:09 smithi148 bash[82222]: audit 2024-03-07T03:54:09.897055+0000 mon.a (mon.0) 2012 : audit 0 from='client.? 172.21.15.148:0/4068973079' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:10.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:09 smithi195 bash[66542]: audit 2024-03-07T03:54:09.897055+0000 mon.a (mon.0) 2012 : audit 0 from='client.? 172.21.15.148:0/4068973079' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:11.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:10 smithi148 bash[79819]: cluster 2024-03-07T03:54:09.355943+0000 mgr.y (mgr.34101) 1208 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:11.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:10 smithi148 bash[82222]: cluster 2024-03-07T03:54:09.355943+0000 mgr.y (mgr.34101) 1208 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:11.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:10 smithi195 bash[66542]: cluster 2024-03-07T03:54:09.355943+0000 mgr.y (mgr.34101) 1208 : cluster 0 pgmap v1927: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:12.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:54:12 smithi195 bash[59134]: ts=2024-03-07T03:54:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:13.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:12 smithi148 bash[79819]: cluster 2024-03-07T03:54:11.356862+0000 mgr.y (mgr.34101) 1209 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:13.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:12 smithi148 bash[79819]: audit 2024-03-07T03:54:12.359794+0000 mon.a (mon.0) 2013 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:13.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:12 smithi148 bash[82222]: cluster 2024-03-07T03:54:11.356862+0000 mgr.y (mgr.34101) 1209 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:13.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:12 smithi148 bash[82222]: audit 2024-03-07T03:54:12.359794+0000 mon.a (mon.0) 2013 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:13.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:12 smithi195 bash[66542]: cluster 2024-03-07T03:54:11.356862+0000 mgr.y (mgr.34101) 1209 : cluster 0 pgmap v1928: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:13.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:12 smithi195 bash[66542]: audit 2024-03-07T03:54:12.359794+0000 mon.a (mon.0) 2013 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:13.617 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:15.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:14 smithi148 bash[79819]: cluster 2024-03-07T03:54:13.357734+0000 mgr.y (mgr.34101) 1210 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:15.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:14 smithi148 bash[82222]: cluster 2024-03-07T03:54:13.357734+0000 mgr.y (mgr.34101) 1210 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:15.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:14 smithi195 bash[66542]: cluster 2024-03-07T03:54:13.357734+0000 mgr.y (mgr.34101) 1210 : cluster 0 pgmap v1929: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:17.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:54:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:54:17] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:54:17.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:16 smithi148 bash[79819]: cluster 2024-03-07T03:54:15.358590+0000 mgr.y (mgr.34101) 1211 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:17.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:16 smithi148 bash[82222]: cluster 2024-03-07T03:54:15.358590+0000 mgr.y (mgr.34101) 1211 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:17.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:16 smithi195 bash[66542]: cluster 2024-03-07T03:54:15.358590+0000 mgr.y (mgr.34101) 1211 : cluster 0 pgmap v1930: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:18.361 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:19.300 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:18 smithi148 bash[79819]: cluster 2024-03-07T03:54:17.359393+0000 mgr.y (mgr.34101) 1212 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:19.300 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:18 smithi148 bash[82222]: cluster 2024-03-07T03:54:17.359393+0000 mgr.y (mgr.34101) 1212 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:19.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:18 smithi195 bash[66542]: cluster 2024-03-07T03:54:17.359393+0000 mgr.y (mgr.34101) 1212 : cluster 0 pgmap v1931: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:20.302 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:19 smithi148 bash[79819]: cluster 2024-03-07T03:54:19.360073+0000 mgr.y (mgr.34101) 1213 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:20.302 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:19 smithi148 bash[82222]: cluster 2024-03-07T03:54:19.360073+0000 mgr.y (mgr.34101) 1213 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:20.323 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:19 smithi195 bash[66542]: cluster 2024-03-07T03:54:19.360073+0000 mgr.y (mgr.34101) 1213 : cluster 0 pgmap v1932: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:21.533 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:54:21.533 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:22.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:22 smithi148 bash[79819]: cluster 2024-03-07T03:54:21.360839+0000 mgr.y (mgr.34101) 1214 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:22.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:22 smithi148 bash[79819]: audit 2024-03-07T03:54:21.534456+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.148:0/2244641232' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:22.552 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:22 smithi148 bash[82222]: cluster 2024-03-07T03:54:21.360839+0000 mgr.y (mgr.34101) 1214 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:22.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:22 smithi148 bash[82222]: audit 2024-03-07T03:54:21.534456+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.148:0/2244641232' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:22.573 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:54:22 smithi195 bash[59134]: ts=2024-03-07T03:54:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:22.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:22 smithi195 bash[66542]: cluster 2024-03-07T03:54:21.360839+0000 mgr.y (mgr.34101) 1214 : cluster 0 pgmap v1933: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:22.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:22 smithi195 bash[66542]: audit 2024-03-07T03:54:21.534456+0000 mon.c (mon.2) 93 : audit 0 from='client.? 172.21.15.148:0/2244641232' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:24.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:24 smithi148 bash[79819]: cluster 2024-03-07T03:54:23.361682+0000 mgr.y (mgr.34101) 1215 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:24.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:24 smithi148 bash[82222]: cluster 2024-03-07T03:54:23.361682+0000 mgr.y (mgr.34101) 1215 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:24.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:24 smithi195 bash[66542]: cluster 2024-03-07T03:54:23.361682+0000 mgr.y (mgr.34101) 1215 : cluster 0 pgmap v1934: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:25.190 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:26.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:26 smithi148 bash[79819]: cluster 2024-03-07T03:54:25.362300+0000 mgr.y (mgr.34101) 1216 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:26.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:26 smithi148 bash[82222]: cluster 2024-03-07T03:54:25.362300+0000 mgr.y (mgr.34101) 1216 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:26.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:26 smithi195 bash[66542]: cluster 2024-03-07T03:54:25.362300+0000 mgr.y (mgr.34101) 1216 : cluster 0 pgmap v1935: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:27.434 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:54:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:54:27] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:54:27.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:27 smithi148 bash[79819]: audit 2024-03-07T03:54:27.360325+0000 mon.a (mon.0) 2014 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:27.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:27 smithi148 bash[82222]: audit 2024-03-07T03:54:27.360325+0000 mon.a (mon.0) 2014 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:27.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:27 smithi195 bash[66542]: audit 2024-03-07T03:54:27.360325+0000 mon.a (mon.0) 2014 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:28.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:28 smithi148 bash[79819]: cluster 2024-03-07T03:54:27.363055+0000 mgr.y (mgr.34101) 1217 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:28.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:28 smithi148 bash[82222]: cluster 2024-03-07T03:54:27.363055+0000 mgr.y (mgr.34101) 1217 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:28.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:28 smithi195 bash[66542]: cluster 2024-03-07T03:54:27.363055+0000 mgr.y (mgr.34101) 1217 : cluster 0 pgmap v1936: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:29.934 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:30.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:30 smithi148 bash[79819]: cluster 2024-03-07T03:54:29.364067+0000 mgr.y (mgr.34101) 1218 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:30.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:30 smithi148 bash[82222]: cluster 2024-03-07T03:54:29.364067+0000 mgr.y (mgr.34101) 1218 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:30.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:30 smithi195 bash[66542]: cluster 2024-03-07T03:54:29.364067+0000 mgr.y (mgr.34101) 1218 : cluster 0 pgmap v1937: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:32.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:32 smithi195 bash[66542]: cluster 2024-03-07T03:54:31.364860+0000 mgr.y (mgr.34101) 1219 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:32.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:54:32 smithi195 bash[59134]: ts=2024-03-07T03:54:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:32.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:32 smithi148 bash[79819]: cluster 2024-03-07T03:54:31.364860+0000 mgr.y (mgr.34101) 1219 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:32.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:32 smithi148 bash[82222]: cluster 2024-03-07T03:54:31.364860+0000 mgr.y (mgr.34101) 1219 : cluster 0 pgmap v1938: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:32.849 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:54:32.849 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:33.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:33 smithi148 bash[79819]: audit 2024-03-07T03:54:32.850533+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.148:0/2277577907' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:33.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:33 smithi148 bash[82222]: audit 2024-03-07T03:54:32.850533+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.148:0/2277577907' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:33.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:33 smithi195 bash[66542]: audit 2024-03-07T03:54:32.850533+0000 mon.c (mon.2) 94 : audit 0 from='client.? 172.21.15.148:0/2277577907' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:34.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:34 smithi148 bash[79819]: cluster 2024-03-07T03:54:33.365724+0000 mgr.y (mgr.34101) 1220 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:34.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:34 smithi148 bash[82222]: cluster 2024-03-07T03:54:33.365724+0000 mgr.y (mgr.34101) 1220 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:34.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:34 smithi195 bash[66542]: cluster 2024-03-07T03:54:33.365724+0000 mgr.y (mgr.34101) 1220 : cluster 0 pgmap v1939: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:36.670 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:36.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:36 smithi148 bash[79819]: cluster 2024-03-07T03:54:35.366631+0000 mgr.y (mgr.34101) 1221 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:36.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:36 smithi148 bash[82222]: cluster 2024-03-07T03:54:35.366631+0000 mgr.y (mgr.34101) 1221 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:36.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:36 smithi195 bash[66542]: cluster 2024-03-07T03:54:35.366631+0000 mgr.y (mgr.34101) 1221 : cluster 0 pgmap v1940: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:37.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:54:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:54:37] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:54:38.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:38 smithi148 bash[79819]: cluster 2024-03-07T03:54:37.367333+0000 mgr.y (mgr.34101) 1222 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:38.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:38 smithi148 bash[82222]: cluster 2024-03-07T03:54:37.367333+0000 mgr.y (mgr.34101) 1222 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:38.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:38 smithi195 bash[66542]: cluster 2024-03-07T03:54:37.367333+0000 mgr.y (mgr.34101) 1222 : cluster 0 pgmap v1941: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:40.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:40 smithi148 bash[79819]: cluster 2024-03-07T03:54:39.368180+0000 mgr.y (mgr.34101) 1223 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:40.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:40 smithi148 bash[79819]: audit 2024-03-07T03:54:39.833108+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:54:40.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:40 smithi148 bash[82222]: cluster 2024-03-07T03:54:39.368180+0000 mgr.y (mgr.34101) 1223 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:40.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:40 smithi148 bash[82222]: audit 2024-03-07T03:54:39.833108+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:54:40.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:40 smithi195 bash[66542]: cluster 2024-03-07T03:54:39.368180+0000 mgr.y (mgr.34101) 1223 : cluster 0 pgmap v1942: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:40.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:40 smithi195 bash[66542]: audit 2024-03-07T03:54:39.833108+0000 mon.a (mon.0) 2015 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:54:41.404 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:42 smithi195 bash[66542]: cluster 2024-03-07T03:54:41.369038+0000 mgr.y (mgr.34101) 1224 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:42.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:42 smithi195 bash[66542]: audit 2024-03-07T03:54:42.360762+0000 mon.a (mon.0) 2016 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:42.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:54:42 smithi195 bash[59134]: ts=2024-03-07T03:54:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:42.712 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:42 smithi148 bash[82222]: cluster 2024-03-07T03:54:41.369038+0000 mgr.y (mgr.34101) 1224 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:42.712 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:42 smithi148 bash[82222]: audit 2024-03-07T03:54:42.360762+0000 mon.a (mon.0) 2016 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:42.712 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:42 smithi148 bash[79819]: cluster 2024-03-07T03:54:41.369038+0000 mgr.y (mgr.34101) 1224 : cluster 0 pgmap v1943: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:42.712 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:42 smithi148 bash[79819]: audit 2024-03-07T03:54:42.360762+0000 mon.a (mon.0) 2016 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:44.444 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:54:44.444 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:44.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:44 smithi148 bash[79819]: cluster 2024-03-07T03:54:43.370031+0000 mgr.y (mgr.34101) 1225 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:44.739 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:44 smithi148 bash[79819]: audit 2024-03-07T03:54:44.445227+0000 mon.a (mon.0) 2017 : audit 0 from='client.? 172.21.15.148:0/3732113629' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:44.740 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:44 smithi148 bash[82222]: cluster 2024-03-07T03:54:43.370031+0000 mgr.y (mgr.34101) 1225 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:44.740 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:44 smithi148 bash[82222]: audit 2024-03-07T03:54:44.445227+0000 mon.a (mon.0) 2017 : audit 0 from='client.? 172.21.15.148:0/3732113629' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:44.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:44 smithi195 bash[66542]: cluster 2024-03-07T03:54:43.370031+0000 mgr.y (mgr.34101) 1225 : cluster 0 pgmap v1944: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:44.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:44 smithi195 bash[66542]: audit 2024-03-07T03:54:44.445227+0000 mon.a (mon.0) 2017 : audit 0 from='client.? 172.21.15.148:0/3732113629' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:46.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[79819]: audit 2024-03-07T03:54:45.179342+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[79819]: audit 2024-03-07T03:54:45.190214+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[79819]: cluster 2024-03-07T03:54:45.370960+0000 mgr.y (mgr.34101) 1226 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:46.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[79819]: audit 2024-03-07T03:54:45.942888+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[79819]: audit 2024-03-07T03:54:45.953598+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[82222]: audit 2024-03-07T03:54:45.179342+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[82222]: audit 2024-03-07T03:54:45.190214+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[82222]: cluster 2024-03-07T03:54:45.370960+0000 mgr.y (mgr.34101) 1226 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:46.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[82222]: audit 2024-03-07T03:54:45.942888+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:46 smithi148 bash[82222]: audit 2024-03-07T03:54:45.953598+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:46 smithi195 bash[66542]: audit 2024-03-07T03:54:45.179342+0000 mon.a (mon.0) 2018 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:46 smithi195 bash[66542]: audit 2024-03-07T03:54:45.190214+0000 mon.a (mon.0) 2019 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:46 smithi195 bash[66542]: cluster 2024-03-07T03:54:45.370960+0000 mgr.y (mgr.34101) 1226 : cluster 0 pgmap v1945: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:46.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:46 smithi195 bash[66542]: audit 2024-03-07T03:54:45.942888+0000 mon.a (mon.0) 2020 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:46.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:46 smithi195 bash[66542]: audit 2024-03-07T03:54:45.953598+0000 mon.a (mon.0) 2021 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:47.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:54:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:54:47] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:54:47.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:47 smithi148 bash[79819]: audit 2024-03-07T03:54:46.237106+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:47.553 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:47 smithi148 bash[79819]: audit 2024-03-07T03:54:46.242667+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:47.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:47 smithi148 bash[82222]: audit 2024-03-07T03:54:46.237106+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:47.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:47 smithi148 bash[82222]: audit 2024-03-07T03:54:46.242667+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:47.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:47 smithi195 bash[66542]: audit 2024-03-07T03:54:46.237106+0000 mon.a (mon.0) 2022 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:47.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:47 smithi195 bash[66542]: audit 2024-03-07T03:54:46.242667+0000 mon.a (mon.0) 2023 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:48.190 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:54:48.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:48 smithi148 bash[79819]: cluster 2024-03-07T03:54:47.371713+0000 mgr.y (mgr.34101) 1227 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:48.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:48 smithi148 bash[82222]: cluster 2024-03-07T03:54:47.371713+0000 mgr.y (mgr.34101) 1227 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:48.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:48 smithi195 bash[66542]: cluster 2024-03-07T03:54:47.371713+0000 mgr.y (mgr.34101) 1227 : cluster 0 pgmap v1946: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:50.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:50 smithi148 bash[79819]: cluster 2024-03-07T03:54:49.372772+0000 mgr.y (mgr.34101) 1228 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:50.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:50 smithi148 bash[82222]: cluster 2024-03-07T03:54:49.372772+0000 mgr.y (mgr.34101) 1228 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:50.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:50 smithi195 bash[66542]: cluster 2024-03-07T03:54:49.372772+0000 mgr.y (mgr.34101) 1228 : cluster 0 pgmap v1947: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:52.259 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:54:52.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:52 smithi148 bash[79819]: cluster 2024-03-07T03:54:51.373633+0000 mgr.y (mgr.34101) 1229 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:52.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:52 smithi148 bash[82222]: cluster 2024-03-07T03:54:51.373633+0000 mgr.y (mgr.34101) 1229 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:52.573 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:52 smithi195 bash[66542]: cluster 2024-03-07T03:54:51.373633+0000 mgr.y (mgr.34101) 1229 : cluster 0 pgmap v1948: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:52.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:54:52 smithi195 bash[59134]: ts=2024-03-07T03:54:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:54:54.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:54 smithi148 bash[79819]: cluster 2024-03-07T03:54:53.374438+0000 mgr.y (mgr.34101) 1230 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:54.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:54 smithi148 bash[82222]: cluster 2024-03-07T03:54:53.374438+0000 mgr.y (mgr.34101) 1230 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:54.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:54 smithi195 bash[66542]: cluster 2024-03-07T03:54:53.374438+0000 mgr.y (mgr.34101) 1230 : cluster 0 pgmap v1949: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:56.709 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:56 smithi148 bash[79819]: cluster 2024-03-07T03:54:55.375044+0000 mgr.y (mgr.34101) 1231 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:56.709 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:56 smithi148 bash[82222]: cluster 2024-03-07T03:54:55.375044+0000 mgr.y (mgr.34101) 1231 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:56.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:56 smithi195 bash[66542]: cluster 2024-03-07T03:54:55.375044+0000 mgr.y (mgr.34101) 1231 : cluster 0 pgmap v1950: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:56.829 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:54:56.829 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:54:57.455 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:54:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:54:57] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.43.0" 2024-03-07T03:54:57.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:57 smithi148 bash[79819]: audit 2024-03-07T03:54:56.830054+0000 mon.a (mon.0) 2024 : audit 0 from='client.? 172.21.15.148:0/2976446687' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:57.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:57 smithi148 bash[79819]: audit 2024-03-07T03:54:57.360991+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:57.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:57 smithi148 bash[82222]: audit 2024-03-07T03:54:56.830054+0000 mon.a (mon.0) 2024 : audit 0 from='client.? 172.21.15.148:0/2976446687' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:57.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:57 smithi148 bash[82222]: audit 2024-03-07T03:54:57.360991+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:57.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:57 smithi195 bash[66542]: audit 2024-03-07T03:54:56.830054+0000 mon.a (mon.0) 2024 : audit 0 from='client.? 172.21.15.148:0/2976446687' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:54:57.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:57 smithi195 bash[66542]: audit 2024-03-07T03:54:57.360991+0000 mon.a (mon.0) 2025 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:54:58.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[79819]: cluster 2024-03-07T03:54:57.375546+0000 mgr.y (mgr.34101) 1232 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:58.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[79819]: audit 2024-03-07T03:54:57.469386+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[79819]: audit 2024-03-07T03:54:57.481353+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[79819]: audit 2024-03-07T03:54:57.484029+0000 mon.a (mon.0) 2028 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:54:58.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[79819]: audit 2024-03-07T03:54:57.485728+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:54:58.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[79819]: audit 2024-03-07T03:54:57.496864+0000 mon.a (mon.0) 2030 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[82222]: cluster 2024-03-07T03:54:57.375546+0000 mgr.y (mgr.34101) 1232 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:58.804 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[82222]: audit 2024-03-07T03:54:57.469386+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.804 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[82222]: audit 2024-03-07T03:54:57.481353+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.804 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[82222]: audit 2024-03-07T03:54:57.484029+0000 mon.a (mon.0) 2028 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:54:58.804 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[82222]: audit 2024-03-07T03:54:57.485728+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:54:58.804 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:54:58 smithi148 bash[82222]: audit 2024-03-07T03:54:57.496864+0000 mon.a (mon.0) 2030 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:58 smithi195 bash[66542]: cluster 2024-03-07T03:54:57.375546+0000 mgr.y (mgr.34101) 1232 : cluster 0 pgmap v1951: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:54:58.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:58 smithi195 bash[66542]: audit 2024-03-07T03:54:57.469386+0000 mon.a (mon.0) 2026 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:58 smithi195 bash[66542]: audit 2024-03-07T03:54:57.481353+0000 mon.a (mon.0) 2027 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:54:58.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:58 smithi195 bash[66542]: audit 2024-03-07T03:54:57.484029+0000 mon.a (mon.0) 2028 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:54:58.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:58 smithi195 bash[66542]: audit 2024-03-07T03:54:57.485728+0000 mon.a (mon.0) 2029 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:54:58.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:54:58 smithi195 bash[66542]: audit 2024-03-07T03:54:57.496864+0000 mon.a (mon.0) 2030 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:55:00.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:00 smithi148 bash[79819]: cluster 2024-03-07T03:54:59.376550+0000 mgr.y (mgr.34101) 1233 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:00.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:00 smithi148 bash[82222]: cluster 2024-03-07T03:54:59.376550+0000 mgr.y (mgr.34101) 1233 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:00.811 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:00.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:00 smithi195 bash[66542]: cluster 2024-03-07T03:54:59.376550+0000 mgr.y (mgr.34101) 1233 : cluster 0 pgmap v1952: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:02.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:02 smithi195 bash[66542]: cluster 2024-03-07T03:55:01.377423+0000 mgr.y (mgr.34101) 1234 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:02.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:55:02 smithi195 bash[59134]: ts=2024-03-07T03:55:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:02.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:02 smithi148 bash[79819]: cluster 2024-03-07T03:55:01.377423+0000 mgr.y (mgr.34101) 1234 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:02.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:02 smithi148 bash[82222]: cluster 2024-03-07T03:55:01.377423+0000 mgr.y (mgr.34101) 1234 : cluster 0 pgmap v1953: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:04.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:04 smithi148 bash[79819]: cluster 2024-03-07T03:55:03.378213+0000 mgr.y (mgr.34101) 1235 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:04.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:04 smithi148 bash[82222]: cluster 2024-03-07T03:55:03.378213+0000 mgr.y (mgr.34101) 1235 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:04.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:04 smithi195 bash[66542]: cluster 2024-03-07T03:55:03.378213+0000 mgr.y (mgr.34101) 1235 : cluster 0 pgmap v1954: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:05.552 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:06.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:06 smithi148 bash[79819]: cluster 2024-03-07T03:55:05.379144+0000 mgr.y (mgr.34101) 1236 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:06.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:06 smithi148 bash[82222]: cluster 2024-03-07T03:55:05.379144+0000 mgr.y (mgr.34101) 1236 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:06.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:06 smithi195 bash[66542]: cluster 2024-03-07T03:55:05.379144+0000 mgr.y (mgr.34101) 1236 : cluster 0 pgmap v1955: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:07.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:55:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:55:07] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.43.0" 2024-03-07T03:55:08.552 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:08 smithi148 bash[79819]: cluster 2024-03-07T03:55:07.379693+0000 mgr.y (mgr.34101) 1237 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:08.553 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:08 smithi148 bash[82222]: cluster 2024-03-07T03:55:07.379693+0000 mgr.y (mgr.34101) 1237 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:08.702 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:55:08.702 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:08.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:08 smithi195 bash[66542]: cluster 2024-03-07T03:55:07.379693+0000 mgr.y (mgr.34101) 1237 : cluster 0 pgmap v1956: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:09.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:09 smithi148 bash[79819]: audit 2024-03-07T03:55:08.703497+0000 mon.a (mon.0) 2031 : audit 0 from='client.? 172.21.15.148:0/709988611' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:09.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:09 smithi148 bash[82222]: audit 2024-03-07T03:55:08.703497+0000 mon.a (mon.0) 2031 : audit 0 from='client.? 172.21.15.148:0/709988611' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:09.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:09 smithi195 bash[66542]: audit 2024-03-07T03:55:08.703497+0000 mon.a (mon.0) 2031 : audit 0 from='client.? 172.21.15.148:0/709988611' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:10.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:10 smithi148 bash[79819]: cluster 2024-03-07T03:55:09.380697+0000 mgr.y (mgr.34101) 1238 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:10.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:10 smithi148 bash[82222]: cluster 2024-03-07T03:55:09.380697+0000 mgr.y (mgr.34101) 1238 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:10.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:10 smithi195 bash[66542]: cluster 2024-03-07T03:55:09.380697+0000 mgr.y (mgr.34101) 1238 : cluster 0 pgmap v1957: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:12.498 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:12.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:12 smithi195 bash[66542]: cluster 2024-03-07T03:55:11.381531+0000 mgr.y (mgr.34101) 1239 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:12.575 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:12 smithi195 bash[66542]: audit 2024-03-07T03:55:12.361517+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:12.576 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:55:12 smithi195 bash[59134]: ts=2024-03-07T03:55:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:12.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:12 smithi148 bash[79819]: cluster 2024-03-07T03:55:11.381531+0000 mgr.y (mgr.34101) 1239 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:12.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:12 smithi148 bash[79819]: audit 2024-03-07T03:55:12.361517+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:12.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:12 smithi148 bash[82222]: cluster 2024-03-07T03:55:11.381531+0000 mgr.y (mgr.34101) 1239 : cluster 0 pgmap v1958: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:12.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:12 smithi148 bash[82222]: audit 2024-03-07T03:55:12.361517+0000 mon.a (mon.0) 2032 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:14.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:14 smithi148 bash[79819]: cluster 2024-03-07T03:55:13.382357+0000 mgr.y (mgr.34101) 1240 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:14.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:14 smithi148 bash[82222]: cluster 2024-03-07T03:55:13.382357+0000 mgr.y (mgr.34101) 1240 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:14.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:14 smithi195 bash[66542]: cluster 2024-03-07T03:55:13.382357+0000 mgr.y (mgr.34101) 1240 : cluster 0 pgmap v1959: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:16.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:16 smithi148 bash[79819]: cluster 2024-03-07T03:55:15.383407+0000 mgr.y (mgr.34101) 1241 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:16.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:16 smithi148 bash[82222]: cluster 2024-03-07T03:55:15.383407+0000 mgr.y (mgr.34101) 1241 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:16.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:16 smithi195 bash[66542]: cluster 2024-03-07T03:55:15.383407+0000 mgr.y (mgr.34101) 1241 : cluster 0 pgmap v1960: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:17.244 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:17.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:55:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:55:17] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.43.0" 2024-03-07T03:55:18.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:18 smithi148 bash[79819]: cluster 2024-03-07T03:55:17.384215+0000 mgr.y (mgr.34101) 1242 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:18.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:18 smithi148 bash[82222]: cluster 2024-03-07T03:55:17.384215+0000 mgr.y (mgr.34101) 1242 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:18.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:18 smithi195 bash[66542]: cluster 2024-03-07T03:55:17.384215+0000 mgr.y (mgr.34101) 1242 : cluster 0 pgmap v1961: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:20.292 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:55:20.292 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:20.802 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:20 smithi148 bash[82222]: cluster 2024-03-07T03:55:19.385242+0000 mgr.y (mgr.34101) 1243 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:20.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:20 smithi148 bash[82222]: audit 2024-03-07T03:55:20.292622+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.148:0/2953330433' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:20.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:20 smithi148 bash[79819]: cluster 2024-03-07T03:55:19.385242+0000 mgr.y (mgr.34101) 1243 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:20.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:20 smithi148 bash[79819]: audit 2024-03-07T03:55:20.292622+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.148:0/2953330433' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:20.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:20 smithi195 bash[66542]: cluster 2024-03-07T03:55:19.385242+0000 mgr.y (mgr.34101) 1243 : cluster 0 pgmap v1962: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:20.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:20 smithi195 bash[66542]: audit 2024-03-07T03:55:20.292622+0000 mon.c (mon.2) 95 : audit 0 from='client.? 172.21.15.148:0/2953330433' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:22.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:22 smithi195 bash[66542]: cluster 2024-03-07T03:55:21.386167+0000 mgr.y (mgr.34101) 1244 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:22.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:55:22 smithi195 bash[59134]: ts=2024-03-07T03:55:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:22.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:22 smithi148 bash[79819]: cluster 2024-03-07T03:55:21.386167+0000 mgr.y (mgr.34101) 1244 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:22.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:22 smithi148 bash[82222]: cluster 2024-03-07T03:55:21.386167+0000 mgr.y (mgr.34101) 1244 : cluster 0 pgmap v1963: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:23.945 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:24.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:24 smithi148 bash[79819]: cluster 2024-03-07T03:55:23.387085+0000 mgr.y (mgr.34101) 1245 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:24.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:24 smithi148 bash[82222]: cluster 2024-03-07T03:55:23.387085+0000 mgr.y (mgr.34101) 1245 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:24.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:24 smithi195 bash[66542]: cluster 2024-03-07T03:55:23.387085+0000 mgr.y (mgr.34101) 1245 : cluster 0 pgmap v1964: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:26.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:26 smithi148 bash[79819]: cluster 2024-03-07T03:55:25.388025+0000 mgr.y (mgr.34101) 1246 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:26.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:26 smithi148 bash[82222]: cluster 2024-03-07T03:55:25.388025+0000 mgr.y (mgr.34101) 1246 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:26.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:26 smithi195 bash[66542]: cluster 2024-03-07T03:55:25.388025+0000 mgr.y (mgr.34101) 1246 : cluster 0 pgmap v1965: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:27.536 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:55:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:55:27] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.43.0" 2024-03-07T03:55:27.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:27 smithi148 bash[79819]: audit 2024-03-07T03:55:27.362174+0000 mon.a (mon.0) 2033 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:27.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:27 smithi148 bash[82222]: audit 2024-03-07T03:55:27.362174+0000 mon.a (mon.0) 2033 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:27.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:27 smithi195 bash[66542]: audit 2024-03-07T03:55:27.362174+0000 mon.a (mon.0) 2033 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:28.680 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:28.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:28 smithi148 bash[79819]: cluster 2024-03-07T03:55:27.388877+0000 mgr.y (mgr.34101) 1247 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:28.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:28 smithi148 bash[82222]: cluster 2024-03-07T03:55:27.388877+0000 mgr.y (mgr.34101) 1247 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:28.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:28 smithi195 bash[66542]: cluster 2024-03-07T03:55:27.388877+0000 mgr.y (mgr.34101) 1247 : cluster 0 pgmap v1966: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:30.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:30 smithi148 bash[79819]: cluster 2024-03-07T03:55:29.389955+0000 mgr.y (mgr.34101) 1248 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:30.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:30 smithi148 bash[82222]: cluster 2024-03-07T03:55:29.389955+0000 mgr.y (mgr.34101) 1248 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:30.823 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:30 smithi195 bash[66542]: cluster 2024-03-07T03:55:29.389955+0000 mgr.y (mgr.34101) 1248 : cluster 0 pgmap v1967: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:31.801 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:55:31.801 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:32.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:32 smithi195 bash[66542]: cluster 2024-03-07T03:55:31.390526+0000 mgr.y (mgr.34101) 1249 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:32.574 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:32 smithi195 bash[66542]: audit 2024-03-07T03:55:31.802162+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.148:0/1896901813' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:32.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:55:32 smithi195 bash[59134]: ts=2024-03-07T03:55:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:32.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:32 smithi148 bash[79819]: cluster 2024-03-07T03:55:31.390526+0000 mgr.y (mgr.34101) 1249 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:32.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:32 smithi148 bash[79819]: audit 2024-03-07T03:55:31.802162+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.148:0/1896901813' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:32.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:32 smithi148 bash[82222]: cluster 2024-03-07T03:55:31.390526+0000 mgr.y (mgr.34101) 1249 : cluster 0 pgmap v1968: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:32.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:32 smithi148 bash[82222]: audit 2024-03-07T03:55:31.802162+0000 mon.c (mon.2) 96 : audit 0 from='client.? 172.21.15.148:0/1896901813' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:34.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:34 smithi195 bash[66542]: cluster 2024-03-07T03:55:33.391293+0000 mgr.y (mgr.34101) 1250 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:35.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:34 smithi148 bash[79819]: cluster 2024-03-07T03:55:33.391293+0000 mgr.y (mgr.34101) 1250 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:35.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:34 smithi148 bash[82222]: cluster 2024-03-07T03:55:33.391293+0000 mgr.y (mgr.34101) 1250 : cluster 0 pgmap v1969: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:35.446 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:36.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:36 smithi195 bash[66542]: cluster 2024-03-07T03:55:35.392245+0000 mgr.y (mgr.34101) 1251 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:37.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:36 smithi148 bash[79819]: cluster 2024-03-07T03:55:35.392245+0000 mgr.y (mgr.34101) 1251 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:37.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:36 smithi148 bash[82222]: cluster 2024-03-07T03:55:35.392245+0000 mgr.y (mgr.34101) 1251 : cluster 0 pgmap v1970: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:37.552 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:55:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:55:37] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:55:38.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:38 smithi195 bash[66542]: cluster 2024-03-07T03:55:37.392940+0000 mgr.y (mgr.34101) 1252 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:39.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:38 smithi148 bash[79819]: cluster 2024-03-07T03:55:37.392940+0000 mgr.y (mgr.34101) 1252 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:39.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:38 smithi148 bash[82222]: cluster 2024-03-07T03:55:37.392940+0000 mgr.y (mgr.34101) 1252 : cluster 0 pgmap v1971: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:40.188 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:40.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:40 smithi195 bash[66542]: cluster 2024-03-07T03:55:39.393964+0000 mgr.y (mgr.34101) 1253 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:41.055 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:40 smithi148 bash[79819]: cluster 2024-03-07T03:55:39.393964+0000 mgr.y (mgr.34101) 1253 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:41.056 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:40 smithi148 bash[82222]: cluster 2024-03-07T03:55:39.393964+0000 mgr.y (mgr.34101) 1253 : cluster 0 pgmap v1972: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:42.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:55:42 smithi195 bash[59134]: ts=2024-03-07T03:55:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:42.831 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:42 smithi148 bash[79819]: cluster 2024-03-07T03:55:41.394808+0000 mgr.y (mgr.34101) 1254 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:42.831 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:42 smithi148 bash[79819]: audit 2024-03-07T03:55:42.362394+0000 mon.a (mon.0) 2034 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:42.832 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:42 smithi148 bash[82222]: cluster 2024-03-07T03:55:41.394808+0000 mgr.y (mgr.34101) 1254 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:42.832 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:42 smithi148 bash[82222]: audit 2024-03-07T03:55:42.362394+0000 mon.a (mon.0) 2034 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:43.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:42 smithi195 bash[66542]: cluster 2024-03-07T03:55:41.394808+0000 mgr.y (mgr.34101) 1254 : cluster 0 pgmap v1973: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:43.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:42 smithi195 bash[66542]: audit 2024-03-07T03:55:42.362394+0000 mon.a (mon.0) 2034 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:43.275 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:55:43.275 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:43.958 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:43 smithi148 bash[79819]: audit 2024-03-07T03:55:43.276048+0000 mon.a (mon.0) 2035 : audit 0 from='client.? 172.21.15.148:0/2013027229' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:43.959 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:43 smithi148 bash[82222]: audit 2024-03-07T03:55:43.276048+0000 mon.a (mon.0) 2035 : audit 0 from='client.? 172.21.15.148:0/2013027229' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:44.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:43 smithi195 bash[66542]: audit 2024-03-07T03:55:43.276048+0000 mon.a (mon.0) 2035 : audit 0 from='client.? 172.21.15.148:0/2013027229' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:45.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:44 smithi148 bash[79819]: cluster 2024-03-07T03:55:43.395533+0000 mgr.y (mgr.34101) 1255 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:45.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:44 smithi148 bash[82222]: cluster 2024-03-07T03:55:43.395533+0000 mgr.y (mgr.34101) 1255 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:45.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:44 smithi195 bash[66542]: cluster 2024-03-07T03:55:43.395533+0000 mgr.y (mgr.34101) 1255 : cluster 0 pgmap v1974: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:46.961 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:46.972 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:46 smithi148 bash[79819]: cluster 2024-03-07T03:55:45.396567+0000 mgr.y (mgr.34101) 1256 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:46.972 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:46 smithi148 bash[82222]: cluster 2024-03-07T03:55:45.396567+0000 mgr.y (mgr.34101) 1256 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:47.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:46 smithi195 bash[66542]: cluster 2024-03-07T03:55:45.396567+0000 mgr.y (mgr.34101) 1256 : cluster 0 pgmap v1975: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:47.302 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:55:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:55:47] "GET /metrics HTTP/1.1" 200 34307 "" "Prometheus/2.43.0" 2024-03-07T03:55:49.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:48 smithi148 bash[79819]: cluster 2024-03-07T03:55:47.397313+0000 mgr.y (mgr.34101) 1257 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:49.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:48 smithi148 bash[82222]: cluster 2024-03-07T03:55:47.397313+0000 mgr.y (mgr.34101) 1257 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:49.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:48 smithi195 bash[66542]: cluster 2024-03-07T03:55:47.397313+0000 mgr.y (mgr.34101) 1257 : cluster 0 pgmap v1976: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:51.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:50 smithi148 bash[79819]: cluster 2024-03-07T03:55:49.398442+0000 mgr.y (mgr.34101) 1258 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:51.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:50 smithi148 bash[82222]: cluster 2024-03-07T03:55:49.398442+0000 mgr.y (mgr.34101) 1258 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:50 smithi195 bash[66542]: cluster 2024-03-07T03:55:49.398442+0000 mgr.y (mgr.34101) 1258 : cluster 0 pgmap v1977: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:51.697 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:55:52.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:55:52 smithi195 bash[59134]: ts=2024-03-07T03:55:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:55:52.959 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:52 smithi148 bash[79819]: cluster 2024-03-07T03:55:51.399429+0000 mgr.y (mgr.34101) 1259 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:52.960 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:52 smithi148 bash[82222]: cluster 2024-03-07T03:55:51.399429+0000 mgr.y (mgr.34101) 1259 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:53.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:52 smithi195 bash[66542]: cluster 2024-03-07T03:55:51.399429+0000 mgr.y (mgr.34101) 1259 : cluster 0 pgmap v1978: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:54.778 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:55:54.778 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:55:55.009 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:54 smithi148 bash[79819]: cluster 2024-03-07T03:55:53.400312+0000 mgr.y (mgr.34101) 1260 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:55.009 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:54 smithi148 bash[82222]: cluster 2024-03-07T03:55:53.400312+0000 mgr.y (mgr.34101) 1260 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:55.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:54 smithi195 bash[66542]: cluster 2024-03-07T03:55:53.400312+0000 mgr.y (mgr.34101) 1260 : cluster 0 pgmap v1979: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:55.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:55 smithi148 bash[79819]: audit 2024-03-07T03:55:54.778738+0000 mon.a (mon.0) 2036 : audit 0 from='client.? 172.21.15.148:0/3227961786' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:55.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:55 smithi148 bash[82222]: audit 2024-03-07T03:55:54.778738+0000 mon.a (mon.0) 2036 : audit 0 from='client.? 172.21.15.148:0/3227961786' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:56.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:55 smithi195 bash[66542]: audit 2024-03-07T03:55:54.778738+0000 mon.a (mon.0) 2036 : audit 0 from='client.? 172.21.15.148:0/3227961786' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:55:57.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:56 smithi148 bash[79819]: cluster 2024-03-07T03:55:55.401202+0000 mgr.y (mgr.34101) 1261 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:57.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:56 smithi148 bash[82222]: cluster 2024-03-07T03:55:55.401202+0000 mgr.y (mgr.34101) 1261 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:57.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:56 smithi195 bash[66542]: cluster 2024-03-07T03:55:55.401202+0000 mgr.y (mgr.34101) 1261 : cluster 0 pgmap v1980: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:57.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:55:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:55:57] "GET /metrics HTTP/1.1" 200 34307 "" "Prometheus/2.43.0" 2024-03-07T03:55:58.052 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:57 smithi148 bash[79819]: audit 2024-03-07T03:55:57.362821+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:58.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:57 smithi148 bash[79819]: audit 2024-03-07T03:55:57.589658+0000 mon.a (mon.0) 2038 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:55:58.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:57 smithi148 bash[82222]: audit 2024-03-07T03:55:57.362821+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:58.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:57 smithi148 bash[82222]: audit 2024-03-07T03:55:57.589658+0000 mon.a (mon.0) 2038 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:55:58.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:57 smithi195 bash[66542]: audit 2024-03-07T03:55:57.362821+0000 mon.a (mon.0) 2037 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:55:58.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:57 smithi195 bash[66542]: audit 2024-03-07T03:55:57.589658+0000 mon.a (mon.0) 2038 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:55:58.527 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:55:58.802 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:55:58 smithi148 bash[79819]: cluster 2024-03-07T03:55:57.401946+0000 mgr.y (mgr.34101) 1262 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:58.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:55:58 smithi148 bash[82222]: cluster 2024-03-07T03:55:57.401946+0000 mgr.y (mgr.34101) 1262 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:55:59.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:55:58 smithi195 bash[66542]: cluster 2024-03-07T03:55:57.401946+0000 mgr.y (mgr.34101) 1262 : cluster 0 pgmap v1981: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:01.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:00 smithi148 bash[79819]: cluster 2024-03-07T03:55:59.403019+0000 mgr.y (mgr.34101) 1263 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:01.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:00 smithi148 bash[82222]: cluster 2024-03-07T03:55:59.403019+0000 mgr.y (mgr.34101) 1263 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:01.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:00 smithi195 bash[66542]: cluster 2024-03-07T03:55:59.403019+0000 mgr.y (mgr.34101) 1263 : cluster 0 pgmap v1982: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:02.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:56:02 smithi195 bash[59134]: ts=2024-03-07T03:56:02.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:56:02.651 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:56:03.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:02 smithi148 bash[79819]: cluster 2024-03-07T03:56:01.403827+0000 mgr.y (mgr.34101) 1264 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:03.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:02 smithi148 bash[82222]: cluster 2024-03-07T03:56:01.403827+0000 mgr.y (mgr.34101) 1264 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:03.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:02 smithi195 bash[66542]: cluster 2024-03-07T03:56:01.403827+0000 mgr.y (mgr.34101) 1264 : cluster 0 pgmap v1983: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:04.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[82222]: cluster 2024-03-07T03:56:03.404407+0000 mgr.y (mgr.34101) 1265 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:04.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[82222]: audit 2024-03-07T03:56:03.429733+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[82222]: audit 2024-03-07T03:56:03.435608+0000 mon.a (mon.0) 2040 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[82222]: audit 2024-03-07T03:56:03.588698+0000 mon.a (mon.0) 2041 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[82222]: audit 2024-03-07T03:56:03.597461+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[79819]: cluster 2024-03-07T03:56:03.404407+0000 mgr.y (mgr.34101) 1265 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:04.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[79819]: audit 2024-03-07T03:56:03.429733+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[79819]: audit 2024-03-07T03:56:03.435608+0000 mon.a (mon.0) 2040 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[79819]: audit 2024-03-07T03:56:03.588698+0000 mon.a (mon.0) 2041 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.804 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:04 smithi148 bash[79819]: audit 2024-03-07T03:56:03.597461+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:04 smithi195 bash[66542]: cluster 2024-03-07T03:56:03.404407+0000 mgr.y (mgr.34101) 1265 : cluster 0 pgmap v1984: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:04.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:04 smithi195 bash[66542]: audit 2024-03-07T03:56:03.429733+0000 mon.a (mon.0) 2039 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:04 smithi195 bash[66542]: audit 2024-03-07T03:56:03.435608+0000 mon.a (mon.0) 2040 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:04 smithi195 bash[66542]: audit 2024-03-07T03:56:03.588698+0000 mon.a (mon.0) 2041 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:04.824 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:04 smithi195 bash[66542]: audit 2024-03-07T03:56:03.597461+0000 mon.a (mon.0) 2042 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:05.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:05 smithi148 bash[79819]: audit 2024-03-07T03:56:04.604705+0000 mon.a (mon.0) 2043 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:05.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:05 smithi148 bash[79819]: audit 2024-03-07T03:56:04.618877+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:05.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:05 smithi148 bash[82222]: audit 2024-03-07T03:56:04.604705+0000 mon.a (mon.0) 2043 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:05.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:05 smithi148 bash[82222]: audit 2024-03-07T03:56:04.618877+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:05.877 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:56:05.877 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:56:06.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:05 smithi195 bash[66542]: audit 2024-03-07T03:56:04.604705+0000 mon.a (mon.0) 2043 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:06.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:05 smithi195 bash[66542]: audit 2024-03-07T03:56:04.618877+0000 mon.a (mon.0) 2044 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:07.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:06 smithi148 bash[79819]: cluster 2024-03-07T03:56:05.405347+0000 mgr.y (mgr.34101) 1266 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:07.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:06 smithi148 bash[79819]: audit 2024-03-07T03:56:05.877631+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.148:0/1605928041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:07.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:06 smithi148 bash[82222]: cluster 2024-03-07T03:56:05.405347+0000 mgr.y (mgr.34101) 1266 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:07.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:06 smithi148 bash[82222]: audit 2024-03-07T03:56:05.877631+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.148:0/1605928041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:07.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:06 smithi195 bash[66542]: cluster 2024-03-07T03:56:05.405347+0000 mgr.y (mgr.34101) 1266 : cluster 0 pgmap v1985: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:07.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:06 smithi195 bash[66542]: audit 2024-03-07T03:56:05.877631+0000 mon.c (mon.2) 97 : audit 0 from='client.? 172.21.15.148:0/1605928041' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:07.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:56:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:56:07] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:56:09.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:08 smithi148 bash[79819]: cluster 2024-03-07T03:56:07.406021+0000 mgr.y (mgr.34101) 1267 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:09.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:08 smithi148 bash[82222]: cluster 2024-03-07T03:56:07.406021+0000 mgr.y (mgr.34101) 1267 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:09.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:08 smithi195 bash[66542]: cluster 2024-03-07T03:56:07.406021+0000 mgr.y (mgr.34101) 1267 : cluster 0 pgmap v1986: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:09.684 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:56:11.027 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:10 smithi195 bash[66542]: cluster 2024-03-07T03:56:09.407066+0000 mgr.y (mgr.34101) 1268 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:11.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:10 smithi148 bash[79819]: cluster 2024-03-07T03:56:09.407066+0000 mgr.y (mgr.34101) 1268 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:11.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:10 smithi148 bash[82222]: cluster 2024-03-07T03:56:09.407066+0000 mgr.y (mgr.34101) 1268 : cluster 0 pgmap v1987: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:12.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:56:12 smithi195 bash[59134]: ts=2024-03-07T03:56:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:56:13.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:12 smithi148 bash[82222]: cluster 2024-03-07T03:56:11.407905+0000 mgr.y (mgr.34101) 1269 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:13.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:12 smithi148 bash[82222]: audit 2024-03-07T03:56:12.363242+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:13.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:12 smithi148 bash[79819]: cluster 2024-03-07T03:56:11.407905+0000 mgr.y (mgr.34101) 1269 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:13.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:12 smithi148 bash[79819]: audit 2024-03-07T03:56:12.363242+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:13.073 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:12 smithi195 bash[66542]: cluster 2024-03-07T03:56:11.407905+0000 mgr.y (mgr.34101) 1269 : cluster 0 pgmap v1988: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:13.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:12 smithi195 bash[66542]: audit 2024-03-07T03:56:12.363242+0000 mon.a (mon.0) 2045 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:14.427 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:56:14.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:14 smithi148 bash[79819]: cluster 2024-03-07T03:56:13.408729+0000 mgr.y (mgr.34101) 1270 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:14.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:14 smithi148 bash[79819]: audit 2024-03-07T03:56:14.626493+0000 mon.a (mon.0) 2046 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:14.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:14 smithi148 bash[82222]: cluster 2024-03-07T03:56:13.408729+0000 mgr.y (mgr.34101) 1270 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:14.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:14 smithi148 bash[82222]: audit 2024-03-07T03:56:14.626493+0000 mon.a (mon.0) 2046 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:15.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:14 smithi195 bash[66542]: cluster 2024-03-07T03:56:13.408729+0000 mgr.y (mgr.34101) 1270 : cluster 0 pgmap v1989: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:15.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:14 smithi195 bash[66542]: audit 2024-03-07T03:56:14.626493+0000 mon.a (mon.0) 2046 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[82222]: audit 2024-03-07T03:56:14.639378+0000 mon.a (mon.0) 2047 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[82222]: audit 2024-03-07T03:56:14.642013+0000 mon.a (mon.0) 2048 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[82222]: audit 2024-03-07T03:56:14.651511+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[82222]: audit 2024-03-07T03:56:14.663863+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[79819]: audit 2024-03-07T03:56:14.639378+0000 mon.a (mon.0) 2047 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[79819]: audit 2024-03-07T03:56:14.642013+0000 mon.a (mon.0) 2048 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[79819]: audit 2024-03-07T03:56:14.651511+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:56:15.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:15 smithi148 bash[79819]: audit 2024-03-07T03:56:14.663863+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:16.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:15 smithi195 bash[66542]: audit 2024-03-07T03:56:14.639378+0000 mon.a (mon.0) 2047 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:16.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:15 smithi195 bash[66542]: audit 2024-03-07T03:56:14.642013+0000 mon.a (mon.0) 2048 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:56:16.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:15 smithi195 bash[66542]: audit 2024-03-07T03:56:14.651511+0000 mon.a (mon.0) 2049 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:56:16.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:15 smithi195 bash[66542]: audit 2024-03-07T03:56:14.663863+0000 mon.a (mon.0) 2050 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:56:16.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:16 smithi148 bash[79819]: cluster 2024-03-07T03:56:15.409652+0000 mgr.y (mgr.34101) 1271 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:16.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:16 smithi148 bash[82222]: cluster 2024-03-07T03:56:15.409652+0000 mgr.y (mgr.34101) 1271 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:17.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:16 smithi195 bash[66542]: cluster 2024-03-07T03:56:15.409652+0000 mgr.y (mgr.34101) 1271 : cluster 0 pgmap v1990: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:17.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:56:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:56:17] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:56:17.654 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:56:17.654 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:56:18.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:17 smithi148 bash[82222]: audit 2024-03-07T03:56:17.655015+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.148:0/1185985139' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:18.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:17 smithi148 bash[79819]: audit 2024-03-07T03:56:17.655015+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.148:0/1185985139' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:18.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:17 smithi195 bash[66542]: audit 2024-03-07T03:56:17.655015+0000 mon.c (mon.2) 98 : audit 0 from='client.? 172.21.15.148:0/1185985139' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:19.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:18 smithi148 bash[79819]: cluster 2024-03-07T03:56:17.410065+0000 mgr.y (mgr.34101) 1272 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:19.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:18 smithi148 bash[82222]: cluster 2024-03-07T03:56:17.410065+0000 mgr.y (mgr.34101) 1272 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:19.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:18 smithi195 bash[66542]: cluster 2024-03-07T03:56:17.410065+0000 mgr.y (mgr.34101) 1272 : cluster 0 pgmap v1991: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:21.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:20 smithi148 bash[79819]: cluster 2024-03-07T03:56:19.411027+0000 mgr.y (mgr.34101) 1273 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:21.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:20 smithi148 bash[82222]: cluster 2024-03-07T03:56:19.411027+0000 mgr.y (mgr.34101) 1273 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:21.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:20 smithi195 bash[66542]: cluster 2024-03-07T03:56:19.411027+0000 mgr.y (mgr.34101) 1273 : cluster 0 pgmap v1992: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:21.407 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:56:22.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:56:22 smithi195 bash[59134]: ts=2024-03-07T03:56:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:56:23.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:22 smithi148 bash[79819]: cluster 2024-03-07T03:56:21.411843+0000 mgr.y (mgr.34101) 1274 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:23.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:22 smithi148 bash[82222]: cluster 2024-03-07T03:56:21.411843+0000 mgr.y (mgr.34101) 1274 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:23.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:22 smithi195 bash[66542]: cluster 2024-03-07T03:56:21.411843+0000 mgr.y (mgr.34101) 1274 : cluster 0 pgmap v1993: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:25.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:24 smithi148 bash[79819]: cluster 2024-03-07T03:56:23.412675+0000 mgr.y (mgr.34101) 1275 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:25.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:24 smithi148 bash[82222]: cluster 2024-03-07T03:56:23.412675+0000 mgr.y (mgr.34101) 1275 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:25.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:24 smithi195 bash[66542]: cluster 2024-03-07T03:56:23.412675+0000 mgr.y (mgr.34101) 1275 : cluster 0 pgmap v1994: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:26.148 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:56:27.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:26 smithi148 bash[79819]: cluster 2024-03-07T03:56:25.413646+0000 mgr.y (mgr.34101) 1276 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:27.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:26 smithi148 bash[82222]: cluster 2024-03-07T03:56:25.413646+0000 mgr.y (mgr.34101) 1276 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:27.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:26 smithi195 bash[66542]: cluster 2024-03-07T03:56:25.413646+0000 mgr.y (mgr.34101) 1276 : cluster 0 pgmap v1995: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:27.438 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:56:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:56:27] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:56:28.039 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:27 smithi148 bash[79819]: audit 2024-03-07T03:56:27.363578+0000 mon.a (mon.0) 2051 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:28.040 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:27 smithi148 bash[82222]: audit 2024-03-07T03:56:27.363578+0000 mon.a (mon.0) 2051 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:28.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:27 smithi195 bash[66542]: audit 2024-03-07T03:56:27.363578+0000 mon.a (mon.0) 2051 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:28.961 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:28 smithi148 bash[79819]: cluster 2024-03-07T03:56:27.414111+0000 mgr.y (mgr.34101) 1277 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:28.961 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:28 smithi148 bash[82222]: cluster 2024-03-07T03:56:27.414111+0000 mgr.y (mgr.34101) 1277 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:29.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:28 smithi195 bash[66542]: cluster 2024-03-07T03:56:27.414111+0000 mgr.y (mgr.34101) 1277 : cluster 0 pgmap v1996: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:29.356 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:56:29.356 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:56:30.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:29 smithi148 bash[79819]: audit 2024-03-07T03:56:29.356970+0000 mon.a (mon.0) 2052 : audit 0 from='client.? 172.21.15.148:0/1033927937' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:30.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:29 smithi148 bash[82222]: audit 2024-03-07T03:56:29.356970+0000 mon.a (mon.0) 2052 : audit 0 from='client.? 172.21.15.148:0/1033927937' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:30.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:29 smithi195 bash[66542]: audit 2024-03-07T03:56:29.356970+0000 mon.a (mon.0) 2052 : audit 0 from='client.? 172.21.15.148:0/1033927937' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:31.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:30 smithi148 bash[79819]: cluster 2024-03-07T03:56:29.414681+0000 mgr.y (mgr.34101) 1278 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:31.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:30 smithi148 bash[82222]: cluster 2024-03-07T03:56:29.414681+0000 mgr.y (mgr.34101) 1278 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:31.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:30 smithi195 bash[66542]: cluster 2024-03-07T03:56:29.414681+0000 mgr.y (mgr.34101) 1278 : cluster 0 pgmap v1997: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:32.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:56:32 smithi195 bash[59134]: ts=2024-03-07T03:56:32.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:56:33.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:32 smithi148 bash[79819]: cluster 2024-03-07T03:56:31.415457+0000 mgr.y (mgr.34101) 1279 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:33.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:32 smithi148 bash[82222]: cluster 2024-03-07T03:56:31.415457+0000 mgr.y (mgr.34101) 1279 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:33.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:32 smithi195 bash[66542]: cluster 2024-03-07T03:56:31.415457+0000 mgr.y (mgr.34101) 1279 : cluster 0 pgmap v1998: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:33.152 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:56:35.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:34 smithi148 bash[79819]: cluster 2024-03-07T03:56:33.416354+0000 mgr.y (mgr.34101) 1280 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:35.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:34 smithi148 bash[82222]: cluster 2024-03-07T03:56:33.416354+0000 mgr.y (mgr.34101) 1280 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:35.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:34 smithi195 bash[66542]: cluster 2024-03-07T03:56:33.416354+0000 mgr.y (mgr.34101) 1280 : cluster 0 pgmap v1999: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:37.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:36 smithi148 bash[79819]: cluster 2024-03-07T03:56:35.417246+0000 mgr.y (mgr.34101) 1281 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:37.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:36 smithi148 bash[82222]: cluster 2024-03-07T03:56:35.417246+0000 mgr.y (mgr.34101) 1281 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:37.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:36 smithi195 bash[66542]: cluster 2024-03-07T03:56:35.417246+0000 mgr.y (mgr.34101) 1281 : cluster 0 pgmap v2000: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:37.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:56:37 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:56:37] "GET /metrics HTTP/1.1" 200 34306 "" "Prometheus/2.43.0" 2024-03-07T03:56:37.891 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:56:39.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:38 smithi148 bash[79819]: cluster 2024-03-07T03:56:37.417927+0000 mgr.y (mgr.34101) 1282 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:39.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:38 smithi148 bash[82222]: cluster 2024-03-07T03:56:37.417927+0000 mgr.y (mgr.34101) 1282 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:39.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:38 smithi195 bash[66542]: cluster 2024-03-07T03:56:37.417927+0000 mgr.y (mgr.34101) 1282 : cluster 0 pgmap v2001: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:40.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:40 smithi148 bash[82222]: cluster 2024-03-07T03:56:39.418950+0000 mgr.y (mgr.34101) 1283 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:40.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:40 smithi148 bash[79819]: cluster 2024-03-07T03:56:39.418950+0000 mgr.y (mgr.34101) 1283 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:40.883 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:56:40.884 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:56:41.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:40 smithi195 bash[66542]: cluster 2024-03-07T03:56:39.418950+0000 mgr.y (mgr.34101) 1283 : cluster 0 pgmap v2002: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:42.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:41 smithi148 bash[79819]: audit 2024-03-07T03:56:40.884329+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.148:0/2237398818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:42.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:41 smithi148 bash[82222]: audit 2024-03-07T03:56:40.884329+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.148:0/2237398818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:42.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:41 smithi195 bash[66542]: audit 2024-03-07T03:56:40.884329+0000 mon.c (mon.2) 99 : audit 0 from='client.? 172.21.15.148:0/2237398818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:42.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:56:42 smithi195 bash[59134]: ts=2024-03-07T03:56:42.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:56:43.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:42 smithi148 bash[82222]: cluster 2024-03-07T03:56:41.419766+0000 mgr.y (mgr.34101) 1284 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:43.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:42 smithi148 bash[82222]: audit 2024-03-07T03:56:42.364134+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:43.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:42 smithi148 bash[79819]: cluster 2024-03-07T03:56:41.419766+0000 mgr.y (mgr.34101) 1284 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:43.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:42 smithi148 bash[79819]: audit 2024-03-07T03:56:42.364134+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:43.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:42 smithi195 bash[66542]: cluster 2024-03-07T03:56:41.419766+0000 mgr.y (mgr.34101) 1284 : cluster 0 pgmap v2003: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:43.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:42 smithi195 bash[66542]: audit 2024-03-07T03:56:42.364134+0000 mon.a (mon.0) 2053 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:44.604 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:56:45.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:44 smithi148 bash[79819]: cluster 2024-03-07T03:56:43.420659+0000 mgr.y (mgr.34101) 1285 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:45.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:44 smithi148 bash[82222]: cluster 2024-03-07T03:56:43.420659+0000 mgr.y (mgr.34101) 1285 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:45.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:44 smithi195 bash[66542]: cluster 2024-03-07T03:56:43.420659+0000 mgr.y (mgr.34101) 1285 : cluster 0 pgmap v2004: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:47.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:46 smithi148 bash[79819]: cluster 2024-03-07T03:56:45.421584+0000 mgr.y (mgr.34101) 1286 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:47.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:46 smithi148 bash[82222]: cluster 2024-03-07T03:56:45.421584+0000 mgr.y (mgr.34101) 1286 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:47.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:46 smithi195 bash[66542]: cluster 2024-03-07T03:56:45.421584+0000 mgr.y (mgr.34101) 1286 : cluster 0 pgmap v2005: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:47.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:56:47 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:56:47] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:56:49.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:48 smithi148 bash[79819]: cluster 2024-03-07T03:56:47.422334+0000 mgr.y (mgr.34101) 1287 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:49.054 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:48 smithi148 bash[82222]: cluster 2024-03-07T03:56:47.422334+0000 mgr.y (mgr.34101) 1287 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:49.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:48 smithi195 bash[66542]: cluster 2024-03-07T03:56:47.422334+0000 mgr.y (mgr.34101) 1287 : cluster 0 pgmap v2006: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:49.343 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:56:50.803 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:50 smithi148 bash[79819]: cluster 2024-03-07T03:56:49.423312+0000 mgr.y (mgr.34101) 1288 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:50.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:50 smithi148 bash[82222]: cluster 2024-03-07T03:56:49.423312+0000 mgr.y (mgr.34101) 1288 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:51.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:50 smithi195 bash[66542]: cluster 2024-03-07T03:56:49.423312+0000 mgr.y (mgr.34101) 1288 : cluster 0 pgmap v2007: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:52.253 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:56:52.254 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:56:52.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:56:52 smithi195 bash[59134]: ts=2024-03-07T03:56:52.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:56:53.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:52 smithi148 bash[79819]: cluster 2024-03-07T03:56:51.424173+0000 mgr.y (mgr.34101) 1289 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:53.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:52 smithi148 bash[79819]: audit 2024-03-07T03:56:52.254008+0000 mon.a (mon.0) 2054 : audit 0 from='client.? 172.21.15.148:0/2769482292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:53.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:52 smithi148 bash[82222]: cluster 2024-03-07T03:56:51.424173+0000 mgr.y (mgr.34101) 1289 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:53.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:52 smithi148 bash[82222]: audit 2024-03-07T03:56:52.254008+0000 mon.a (mon.0) 2054 : audit 0 from='client.? 172.21.15.148:0/2769482292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:53.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:52 smithi195 bash[66542]: cluster 2024-03-07T03:56:51.424173+0000 mgr.y (mgr.34101) 1289 : cluster 0 pgmap v2008: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:53.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:52 smithi195 bash[66542]: audit 2024-03-07T03:56:52.254008+0000 mon.a (mon.0) 2054 : audit 0 from='client.? 172.21.15.148:0/2769482292' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:56:55.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:54 smithi148 bash[79819]: cluster 2024-03-07T03:56:53.424987+0000 mgr.y (mgr.34101) 1290 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:55.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:54 smithi148 bash[82222]: cluster 2024-03-07T03:56:53.424987+0000 mgr.y (mgr.34101) 1290 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:55.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:54 smithi195 bash[66542]: cluster 2024-03-07T03:56:53.424987+0000 mgr.y (mgr.34101) 1290 : cluster 0 pgmap v2009: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:55.925 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:56:57.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:56 smithi148 bash[79819]: cluster 2024-03-07T03:56:55.425937+0000 mgr.y (mgr.34101) 1291 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:57.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:56 smithi148 bash[82222]: cluster 2024-03-07T03:56:55.425937+0000 mgr.y (mgr.34101) 1291 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:57.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:56 smithi195 bash[66542]: cluster 2024-03-07T03:56:55.425937+0000 mgr.y (mgr.34101) 1291 : cluster 0 pgmap v2010: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:57.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:56:57 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:56:57] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:56:58.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:57 smithi148 bash[79819]: audit 2024-03-07T03:56:57.364550+0000 mon.a (mon.0) 2055 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:58.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:57 smithi148 bash[82222]: audit 2024-03-07T03:56:57.364550+0000 mon.a (mon.0) 2055 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:58.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:57 smithi195 bash[66542]: audit 2024-03-07T03:56:57.364550+0000 mon.a (mon.0) 2055 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:56:59.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:56:58 smithi148 bash[79819]: cluster 2024-03-07T03:56:57.426747+0000 mgr.y (mgr.34101) 1292 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:59.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:56:58 smithi148 bash[82222]: cluster 2024-03-07T03:56:57.426747+0000 mgr.y (mgr.34101) 1292 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:56:59.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:56:58 smithi195 bash[66542]: cluster 2024-03-07T03:56:57.426747+0000 mgr.y (mgr.34101) 1292 : cluster 0 pgmap v2011: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:00.664 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:57:01.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:00 smithi148 bash[79819]: cluster 2024-03-07T03:56:59.427906+0000 mgr.y (mgr.34101) 1293 : cluster 0 pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:01.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:00 smithi148 bash[82222]: cluster 2024-03-07T03:56:59.427906+0000 mgr.y (mgr.34101) 1293 : cluster 0 pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:01.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:00 smithi195 bash[66542]: cluster 2024-03-07T03:56:59.427906+0000 mgr.y (mgr.34101) 1293 : cluster 0 pgmap v2012: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:02.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:02 smithi195 bash[59134]: ts=2024-03-07T03:57:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:57:03.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:02 smithi148 bash[79819]: cluster 2024-03-07T03:57:01.428648+0000 mgr.y (mgr.34101) 1294 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:03.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:02 smithi148 bash[82222]: cluster 2024-03-07T03:57:01.428648+0000 mgr.y (mgr.34101) 1294 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:03.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:02 smithi195 bash[66542]: cluster 2024-03-07T03:57:01.428648+0000 mgr.y (mgr.34101) 1294 : cluster 0 pgmap v2013: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:03.741 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:57:03.741 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:57:04.053 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:03 smithi148 bash[79819]: audit 2024-03-07T03:57:03.741063+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.148:0/1148926173' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:04.053 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:03 smithi148 bash[82222]: audit 2024-03-07T03:57:03.741063+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.148:0/1148926173' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:04.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:03 smithi195 bash[66542]: audit 2024-03-07T03:57:03.741063+0000 mon.c (mon.2) 100 : audit 0 from='client.? 172.21.15.148:0/1148926173' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:05.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:04 smithi195 bash[66542]: cluster 2024-03-07T03:57:03.429101+0000 mgr.y (mgr.34101) 1295 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:05.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:04 smithi148 bash[79819]: cluster 2024-03-07T03:57:03.429101+0000 mgr.y (mgr.34101) 1295 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:05.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:04 smithi148 bash[82222]: cluster 2024-03-07T03:57:03.429101+0000 mgr.y (mgr.34101) 1295 : cluster 0 pgmap v2014: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:07.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:06 smithi195 bash[66542]: cluster 2024-03-07T03:57:05.430044+0000 mgr.y (mgr.34101) 1296 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:07.179 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:06 smithi148 bash[82222]: cluster 2024-03-07T03:57:05.430044+0000 mgr.y (mgr.34101) 1296 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:07.179 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:06 smithi148 bash[79819]: cluster 2024-03-07T03:57:05.430044+0000 mgr.y (mgr.34101) 1296 : cluster 0 pgmap v2015: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:07.540 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:57:07.551 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:07 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:57:07] "GET /metrics HTTP/1.1" 200 34309 "" "Prometheus/2.43.0" 2024-03-07T03:57:09.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:08 smithi195 bash[66542]: cluster 2024-03-07T03:57:07.430766+0000 mgr.y (mgr.34101) 1297 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:09.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:08 smithi148 bash[79819]: cluster 2024-03-07T03:57:07.430766+0000 mgr.y (mgr.34101) 1297 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:09.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:08 smithi148 bash[82222]: cluster 2024-03-07T03:57:07.430766+0000 mgr.y (mgr.34101) 1297 : cluster 0 pgmap v2016: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:11.074 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:10 smithi195 bash[66542]: cluster 2024-03-07T03:57:09.431880+0000 mgr.y (mgr.34101) 1298 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:11.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:10 smithi148 bash[79819]: cluster 2024-03-07T03:57:09.431880+0000 mgr.y (mgr.34101) 1298 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:11.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:10 smithi148 bash[82222]: cluster 2024-03-07T03:57:09.431880+0000 mgr.y (mgr.34101) 1298 : cluster 0 pgmap v2017: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:12.279 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:57:12.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:12 smithi195 bash[59134]: ts=2024-03-07T03:57:12.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:57:13.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:12 smithi148 bash[79819]: cluster 2024-03-07T03:57:11.432845+0000 mgr.y (mgr.34101) 1299 : cluster 0 pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:13.258 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:12 smithi148 bash[79819]: audit 2024-03-07T03:57:12.365079+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:57:13.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:12 smithi148 bash[82222]: cluster 2024-03-07T03:57:11.432845+0000 mgr.y (mgr.34101) 1299 : cluster 0 pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:13.258 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:12 smithi148 bash[82222]: audit 2024-03-07T03:57:12.365079+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:57:13.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:12 smithi195 bash[66542]: cluster 2024-03-07T03:57:11.432845+0000 mgr.y (mgr.34101) 1299 : cluster 0 pgmap v2018: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:13.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:12 smithi195 bash[66542]: audit 2024-03-07T03:57:12.365079+0000 mon.a (mon.0) 2056 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:57:15.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:14 smithi148 bash[79819]: cluster 2024-03-07T03:57:13.433637+0000 mgr.y (mgr.34101) 1300 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:15.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:14 smithi148 bash[79819]: audit 2024-03-07T03:57:14.756992+0000 mon.a (mon.0) 2057 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:57:15.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:14 smithi148 bash[82222]: cluster 2024-03-07T03:57:13.433637+0000 mgr.y (mgr.34101) 1300 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:15.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:14 smithi148 bash[82222]: audit 2024-03-07T03:57:14.756992+0000 mon.a (mon.0) 2057 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:57:15.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:14 smithi195 bash[66542]: cluster 2024-03-07T03:57:13.433637+0000 mgr.y (mgr.34101) 1300 : cluster 0 pgmap v2019: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:15.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:14 smithi195 bash[66542]: audit 2024-03-07T03:57:14.756992+0000 mon.a (mon.0) 2057 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-07T03:57:15.436 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:57:15.436 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:57:16.229 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:15 smithi148 bash[79819]: audit 2024-03-07T03:57:15.436044+0000 mon.a (mon.0) 2058 : audit 0 from='client.? 172.21.15.148:0/3881167426' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:16.230 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:15 smithi148 bash[82222]: audit 2024-03-07T03:57:15.436044+0000 mon.a (mon.0) 2058 : audit 0 from='client.? 172.21.15.148:0/3881167426' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:16.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:15 smithi195 bash[66542]: audit 2024-03-07T03:57:15.436044+0000 mon.a (mon.0) 2058 : audit 0 from='client.? 172.21.15.148:0/3881167426' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:17.178 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:16 smithi148 bash[79819]: cluster 2024-03-07T03:57:15.434482+0000 mgr.y (mgr.34101) 1301 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:17.179 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:16 smithi148 bash[82222]: cluster 2024-03-07T03:57:15.434482+0000 mgr.y (mgr.34101) 1301 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:17.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:16 smithi195 bash[66542]: cluster 2024-03-07T03:57:15.434482+0000 mgr.y (mgr.34101) 1301 : cluster 0 pgmap v2020: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:17.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:17 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:57:17] "GET /metrics HTTP/1.1" 200 34307 "" "Prometheus/2.43.0" 2024-03-07T03:57:19.232 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 -- ceph health --format=json 2024-03-07T03:57:19.244 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:18 smithi148 bash[79819]: cluster 2024-03-07T03:57:17.435219+0000 mgr.y (mgr.34101) 1302 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:19.244 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:18 smithi148 bash[82222]: cluster 2024-03-07T03:57:17.435219+0000 mgr.y (mgr.34101) 1302 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:19.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:18 smithi195 bash[66542]: cluster 2024-03-07T03:57:17.435219+0000 mgr.y (mgr.34101) 1302 : cluster 0 pgmap v2021: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:21.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:20 smithi148 bash[79819]: cluster 2024-03-07T03:57:19.436231+0000 mgr.y (mgr.34101) 1303 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:21.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:20 smithi148 bash[79819]: audit 2024-03-07T03:57:20.600484+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:21.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:20 smithi148 bash[79819]: audit 2024-03-07T03:57:20.609637+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:21.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:20 smithi148 bash[82222]: cluster 2024-03-07T03:57:19.436231+0000 mgr.y (mgr.34101) 1303 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:21.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:20 smithi148 bash[82222]: audit 2024-03-07T03:57:20.600484+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:21.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:20 smithi148 bash[82222]: audit 2024-03-07T03:57:20.609637+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:21.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:20 smithi195 bash[66542]: cluster 2024-03-07T03:57:19.436231+0000 mgr.y (mgr.34101) 1303 : cluster 0 pgmap v2022: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:21.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:20 smithi195 bash[66542]: audit 2024-03-07T03:57:20.600484+0000 mon.a (mon.0) 2059 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:21.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:20 smithi195 bash[66542]: audit 2024-03-07T03:57:20.609637+0000 mon.a (mon.0) 2060 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[79819]: audit 2024-03-07T03:57:20.864290+0000 mon.a (mon.0) 2061 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[79819]: audit 2024-03-07T03:57:20.877926+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[79819]: audit 2024-03-07T03:57:21.509149+0000 mon.a (mon.0) 2063 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:57:22.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[79819]: audit 2024-03-07T03:57:21.510991+0000 mon.a (mon.0) 2064 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:57:22.304 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[79819]: audit 2024-03-07T03:57:21.521817+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[82222]: audit 2024-03-07T03:57:20.864290+0000 mon.a (mon.0) 2061 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[82222]: audit 2024-03-07T03:57:20.877926+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[82222]: audit 2024-03-07T03:57:21.509149+0000 mon.a (mon.0) 2063 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:57:22.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[82222]: audit 2024-03-07T03:57:21.510991+0000 mon.a (mon.0) 2064 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:57:22.304 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:21 smithi148 bash[82222]: audit 2024-03-07T03:57:21.521817+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:21 smithi195 bash[66542]: audit 2024-03-07T03:57:20.864290+0000 mon.a (mon.0) 2061 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:21 smithi195 bash[66542]: audit 2024-03-07T03:57:20.877926+0000 mon.a (mon.0) 2062 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:21 smithi195 bash[66542]: audit 2024-03-07T03:57:21.509149+0000 mon.a (mon.0) 2063 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-07T03:57:22.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:21 smithi195 bash[66542]: audit 2024-03-07T03:57:21.510991+0000 mon.a (mon.0) 2064 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-07T03:57:22.316 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:21 smithi195 bash[66542]: audit 2024-03-07T03:57:21.521817+0000 mon.a (mon.0) 2065 : audit 1 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' 2024-03-07T03:57:22.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:22 smithi195 bash[59134]: ts=2024-03-07T03:57:22.316Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:57:23.303 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:22 smithi148 bash[79819]: cluster 2024-03-07T03:57:21.436907+0000 mgr.y (mgr.34101) 1304 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:23.303 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:22 smithi148 bash[82222]: cluster 2024-03-07T03:57:21.436907+0000 mgr.y (mgr.34101) 1304 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:23.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:22 smithi195 bash[66542]: cluster 2024-03-07T03:57:21.436907+0000 mgr.y (mgr.34101) 1304 : cluster 0 pgmap v2023: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:23.910 INFO:teuthology.orchestra.run.smithi148.stderr:Inferring config /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/mon.a/config 2024-03-07T03:57:25.262 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:24 smithi148 bash[79819]: cluster 2024-03-07T03:57:23.437609+0000 mgr.y (mgr.34101) 1305 : cluster 0 pgmap v2024: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:25.262 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:24 smithi148 bash[82222]: cluster 2024-03-07T03:57:23.437609+0000 mgr.y (mgr.34101) 1305 : cluster 0 pgmap v2024: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:25.324 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:24 smithi195 bash[66542]: cluster 2024-03-07T03:57:23.437609+0000 mgr.y (mgr.34101) 1305 : cluster 0 pgmap v2024: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:27.042 INFO:teuthology.orchestra.run.smithi148.stdout: 2024-03-07T03:57:27.042 INFO:teuthology.orchestra.run.smithi148.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-03-07T03:57:27.177 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:26 smithi148 bash[79819]: cluster 2024-03-07T03:57:25.438518+0000 mgr.y (mgr.34101) 1306 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:27.177 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:26 smithi148 bash[82222]: cluster 2024-03-07T03:57:25.438518+0000 mgr.y (mgr.34101) 1306 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:27.325 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:26 smithi195 bash[66542]: cluster 2024-03-07T03:57:25.438518+0000 mgr.y (mgr.34101) 1306 : cluster 0 pgmap v2025: 73 pgs: 73 active+clean; 582 KiB data, 704 MiB used, 715 GiB / 715 GiB avail 2024-03-07T03:57:27.553 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:27 smithi148 bash[72584]: ::ffff:172.21.15.195 - - [07/Mar/2024:03:57:27] "GET /metrics HTTP/1.1" 200 34307 "" "Prometheus/2.43.0" 2024-03-07T03:57:27.809 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph_manager.py", line 3214, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-07T03:57:28.070 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e6521ef688a34be887461dfa2bcae1a1 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/github.com_ceph_ceph_ac30ee6cd9cb289983b4db31a5b462d33a1efd30/qa/tasks/ceph_manager.py", line 3214, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-07T03:57:28.072 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-07T03:57:28.082 INFO:tasks.cephadm:Teardown begin 2024-03-07T03:57:28.082 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:57:28.097 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:27 smithi148 bash[79819]: audit 2024-03-07T03:57:27.042670+0000 mon.a (mon.0) 2066 : audit 0 from='client.? 172.21.15.148:0/3169435913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:28.097 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:27 smithi148 bash[79819]: audit 2024-03-07T03:57:27.365583+0000 mon.a (mon.0) 2067 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:57:28.097 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:27 smithi148 bash[82222]: audit 2024-03-07T03:57:27.042670+0000 mon.a (mon.0) 2066 : audit 0 from='client.? 172.21.15.148:0/3169435913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:28.097 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:27 smithi148 bash[82222]: audit 2024-03-07T03:57:27.365583+0000 mon.a (mon.0) 2067 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:57:28.099 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:57:28.114 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-07T03:57:28.114 DEBUG:teuthology.orchestra.run.smithi148:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-07T03:57:28.149 DEBUG:teuthology.orchestra.run.smithi195:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-07T03:57:28.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:27 smithi195 bash[66542]: audit 2024-03-07T03:57:27.042670+0000 mon.a (mon.0) 2066 : audit 0 from='client.? 172.21.15.148:0/3169435913' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-07T03:57:28.150 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:27 smithi195 bash[66542]: audit 2024-03-07T03:57:27.365583+0000 mon.a (mon.0) 2067 : audit 0 from='mgr.34101 172.21.15.148:0/3433048901' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-07T03:57:28.155 INFO:tasks.cephadm:Stopping all daemons... 2024-03-07T03:57:28.155 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-03-07T03:57:28.155 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a 2024-03-07T03:57:28.358 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:28 smithi148 systemd[1]: Stopping Ceph mon.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:28.359 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[79819]: debug 2024-03-07T03:57:28.338+0000 7f195904b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:57:28.359 INFO:journalctl@ceph.mon.a.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[79819]: debug 2024-03-07T03:57:28.338+0000 7f195904b700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-03-07T03:57:28.472 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[72584]: [07/Mar/2024:03:57:28] ENGINE Bus STOPPING 2024-03-07T03:57:28.472 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[72584]: [07/Mar/2024:03:57:28] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-07T03:57:28.473 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[72584]: [07/Mar/2024:03:57:28] ENGINE Bus STOPPED 2024-03-07T03:57:28.473 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[72584]: [07/Mar/2024:03:57:28] ENGINE Bus STARTING 2024-03-07T03:57:28.473 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[72584]: [07/Mar/2024:03:57:28] ENGINE Serving on http://:::9283 2024-03-07T03:57:28.803 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:28 smithi148 bash[72584]: [07/Mar/2024:03:57:28] ENGINE Bus STARTED 2024-03-07T03:57:29.417 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.a.service' 2024-03-07T03:57:29.448 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:29.448 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-03-07T03:57:29.449 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-03-07T03:57:29.449 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.c 2024-03-07T03:57:29.803 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:29 smithi148 systemd[1]: Stopping Ceph mon.c for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:29.804 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:29 smithi148 bash[82222]: debug 2024-03-07T03:57:29.606+0000 7f3fae235700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:57:29.804 INFO:journalctl@ceph.mon.c.smithi148.stdout:Mar 07 03:57:29 smithi148 bash[82222]: debug 2024-03-07T03:57:29.606+0000 7f3fae235700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-03-07T03:57:30.653 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.c.service' 2024-03-07T03:57:30.679 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:30.680 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-03-07T03:57:30.680 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-03-07T03:57:30.680 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.b 2024-03-07T03:57:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:30 smithi195 systemd[1]: Stopping Ceph mon.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:31.019 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:30 smithi195 bash[66542]: debug 2024-03-07T03:57:30.932+0000 7f6354714700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-03-07T03:57:31.020 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:30 smithi195 bash[66542]: debug 2024-03-07T03:57:30.932+0000 7f6354714700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-03-07T03:57:32.025 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.b.service' 2024-03-07T03:57:32.034 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:31 smithi195 bash[89690]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-b 2024-03-07T03:57:32.034 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:32 smithi195 bash[89751]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mon-b 2024-03-07T03:57:32.034 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:32 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mon.b.service: Deactivated successfully. 2024-03-07T03:57:32.034 INFO:journalctl@ceph.mon.b.smithi195.stdout:Mar 07 03:57:32 smithi195 systemd[1]: Stopped Ceph mon.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:57:32.055 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:32.055 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-03-07T03:57:32.055 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-03-07T03:57:32.055 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y 2024-03-07T03:57:32.411 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:32 smithi148 systemd[1]: Stopping Ceph mgr.y for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:32.412 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:32 smithi148 bash[72584]: debug 2024-03-07T03:57:32.202+0000 7f6cf8ab7700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:57:32.412 INFO:journalctl@ceph.mgr.y.smithi148.stdout:Mar 07 03:57:32 smithi148 bash[72584]: debug 2024-03-07T03:57:32.202+0000 7f6cf8ab7700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-03-07T03:57:32.574 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:32 smithi195 bash[59134]: ts=2024-03-07T03:57:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi148:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"b9c09ac8-dc2d-11ee-95c6-87774f69a715\", instance=\"smithi148:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-07T03:57:32.575 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:32 smithi195 bash[59134]: ts=2024-03-07T03:57:32.484Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:57:32.575 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:32 smithi195 bash[59134]: ts=2024-03-07T03:57:32.488Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:57:32.575 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:32 smithi195 bash[59134]: ts=2024-03-07T03:57:32.491Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:57:32.575 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:32 smithi195 bash[59134]: ts=2024-03-07T03:57:32.496Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:57:32.575 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:57:32 smithi195 bash[59134]: ts=2024-03-07T03:57:32.498Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:57:33.022 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.y.service' 2024-03-07T03:57:33.044 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:33.045 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-03-07T03:57:33.045 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-03-07T03:57:33.045 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.x 2024-03-07T03:57:33.324 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:57:33 smithi195 systemd[1]: Stopping Ceph mgr.x for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:34.071 INFO:journalctl@ceph.mgr.x.smithi195.stdout:Mar 07 03:57:33 smithi195 bash[89791]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-mgr-x 2024-03-07T03:57:34.089 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@mgr.x.service' 2024-03-07T03:57:34.113 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:34.113 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-03-07T03:57:34.113 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-07T03:57:34.113 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.0 2024-03-07T03:57:34.553 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:57:34 smithi148 systemd[1]: Stopping Ceph osd.0 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:34.554 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:57:34 smithi148 bash[88937]: debug 2024-03-07T03:57:34.266+0000 7fb50ba48700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:57:34.554 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:57:34 smithi148 bash[88937]: debug 2024-03-07T03:57:34.266+0000 7fb50ba48700 -1 osd.0 1102 *** Got signal Terminated *** 2024-03-07T03:57:34.554 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:57:34 smithi148 bash[88937]: debug 2024-03-07T03:57:34.266+0000 7fb50ba48700 -1 osd.0 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:57:40.536 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:57:40 smithi148 bash[177206]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:57:40.536 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:40 smithi148 bash[58229]: ts=2024-03-07T03:57:40.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:57:40.537 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:40 smithi148 bash[58229]: ts=2024-03-07T03:57:40.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:57:40.803 INFO:journalctl@ceph.osd.0.smithi148.stdout:Mar 07 03:57:40 smithi148 bash[177270]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-0 2024-03-07T03:57:42.412 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.0.service' 2024-03-07T03:57:42.439 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:42.440 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-07T03:57:42.440 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-07T03:57:42.440 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.1 2024-03-07T03:57:42.618 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:57:42 smithi148 systemd[1]: Stopping Ceph osd.1 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:42.803 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:57:42 smithi148 bash[93712]: debug 2024-03-07T03:57:42.614+0000 7fc9f44db700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:57:42.803 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:57:42 smithi148 bash[93712]: debug 2024-03-07T03:57:42.614+0000 7fc9f44db700 -1 osd.1 1102 *** Got signal Terminated *** 2024-03-07T03:57:42.804 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:57:42 smithi148 bash[93712]: debug 2024-03-07T03:57:42.614+0000 7fc9f44db700 -1 osd.1 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:57:43.803 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:43 smithi148 bash[58229]: ts=2024-03-07T03:57:43.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:57:43.803 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:43 smithi148 bash[58229]: ts=2024-03-07T03:57:43.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:57:49.104 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:57:48 smithi148 bash[177406]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:57:49.104 INFO:journalctl@ceph.osd.1.smithi148.stdout:Mar 07 03:57:49 smithi148 bash[177468]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-1 2024-03-07T03:57:50.553 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:50 smithi148 bash[58229]: ts=2024-03-07T03:57:50.268Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:57:50.553 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:50 smithi148 bash[58229]: ts=2024-03-07T03:57:50.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:57:50.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:50 smithi148 bash[58229]: ts=2024-03-07T03:57:50.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:57:50.921 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.1.service' 2024-03-07T03:57:50.949 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:50.949 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-07T03:57:50.949 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-07T03:57:50.949 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.2 2024-03-07T03:57:51.303 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:57:51 smithi148 systemd[1]: Stopping Ceph osd.2 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:51.304 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:57:51 smithi148 bash[99646]: debug 2024-03-07T03:57:51.122+0000 7f73d7180700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:57:51.304 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:57:51 smithi148 bash[99646]: debug 2024-03-07T03:57:51.122+0000 7f73d7180700 -1 osd.2 1102 *** Got signal Terminated *** 2024-03-07T03:57:51.304 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:57:51 smithi148 bash[99646]: debug 2024-03-07T03:57:51.122+0000 7f73d7180700 -1 osd.2 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:57:53.803 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:53 smithi148 bash[58229]: ts=2024-03-07T03:57:53.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:57:53.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:53 smithi148 bash[58229]: ts=2024-03-07T03:57:53.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:57:53.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:57:53 smithi148 bash[58229]: ts=2024-03-07T03:57:53.522Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:57:57.553 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:57:57 smithi148 bash[177602]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:57:57.553 INFO:journalctl@ceph.osd.2.smithi148.stdout:Mar 07 03:57:57 smithi148 bash[177664]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-2 2024-03-07T03:57:59.063 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.2.service' 2024-03-07T03:57:59.089 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:57:59.090 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-07T03:57:59.090 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-07T03:57:59.090 DEBUG:teuthology.orchestra.run.smithi148:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.3 2024-03-07T03:57:59.553 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:57:59 smithi148 systemd[1]: Stopping Ceph osd.3 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:57:59.554 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:57:59 smithi148 bash[104302]: debug 2024-03-07T03:57:59.254+0000 7f62fd10e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:57:59.554 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:57:59 smithi148 bash[104302]: debug 2024-03-07T03:57:59.254+0000 7f62fd10e700 -1 osd.3 1102 *** Got signal Terminated *** 2024-03-07T03:57:59.554 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:57:59 smithi148 bash[104302]: debug 2024-03-07T03:57:59.254+0000 7f62fd10e700 -1 osd.3 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:58:00.553 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:00 smithi148 bash[58229]: ts=2024-03-07T03:58:00.269Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:00.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:00 smithi148 bash[58229]: ts=2024-03-07T03:58:00.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:00.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:00 smithi148 bash[58229]: ts=2024-03-07T03:58:00.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:01.574 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:01 smithi195 bash[72218]: debug 2024-03-07T03:58:01.255+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:01.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:01 smithi195 bash[32082]: debug 2024-03-07T03:58:01.335+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:01.575 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:01 smithi195 bash[37711]: debug 2024-03-07T03:58:01.499+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:02.574 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:02 smithi195 bash[72218]: debug 2024-03-07T03:58:02.259+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:02.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:02 smithi195 bash[32082]: debug 2024-03-07T03:58:02.303+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:02.575 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:02 smithi195 bash[37711]: debug 2024-03-07T03:58:02.483+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:03.493 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:03 smithi195 bash[72218]: debug 2024-03-07T03:58:03.211+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:03.494 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:03 smithi195 bash[32082]: debug 2024-03-07T03:58:03.295+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:03.803 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:03 smithi148 bash[58229]: ts=2024-03-07T03:58:03.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:03.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:03 smithi148 bash[58229]: ts=2024-03-07T03:58:03.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:03.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:03 smithi148 bash[58229]: ts=2024-03-07T03:58:03.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:03.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:03 smithi195 bash[37711]: debug 2024-03-07T03:58:03.491+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:04.553 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:58:04 smithi148 bash[104302]: debug 2024-03-07T03:58:04.106+0000 7f62f56fa700 -1 osd.3 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:39.029626+0000 front 2024-03-07T03:57:39.029507+0000 (oldest deadline 2024-03-07T03:58:03.729544+0000) 2024-03-07T03:58:04.574 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:04 smithi195 bash[72218]: debug 2024-03-07T03:58:04.251+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:04.574 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:04 smithi195 bash[32082]: debug 2024-03-07T03:58:04.279+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:04.575 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:04 smithi195 bash[37711]: debug 2024-03-07T03:58:04.475+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:05.521 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:05 smithi195 bash[72218]: debug 2024-03-07T03:58:05.247+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:05.521 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:05 smithi195 bash[32082]: debug 2024-03-07T03:58:05.311+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:05.803 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:58:05 smithi148 bash[177798]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:58:05.804 INFO:journalctl@ceph.osd.3.smithi148.stdout:Mar 07 03:58:05 smithi148 bash[177864]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-3 2024-03-07T03:58:05.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:05 smithi195 bash[37711]: debug 2024-03-07T03:58:05.519+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:05.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:05 smithi195 bash[43128]: debug 2024-03-07T03:58:05.719+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:06.541 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:06 smithi195 bash[72218]: debug 2024-03-07T03:58:06.295+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:06.542 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:06 smithi195 bash[32082]: debug 2024-03-07T03:58:06.275+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:06.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:06 smithi195 bash[37711]: debug 2024-03-07T03:58:06.539+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:06.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:06 smithi195 bash[43128]: debug 2024-03-07T03:58:06.679+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:07.229 DEBUG:teuthology.orchestra.run.smithi148:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.3.service' 2024-03-07T03:58:07.256 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:58:07.257 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-07T03:58:07.257 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-07T03:58:07.257 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.4 2024-03-07T03:58:07.564 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:07 smithi195 bash[32082]: debug 2024-03-07T03:58:07.271+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:07.565 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:07 smithi195 systemd[1]: Stopping Ceph osd.4 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:58:07.565 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:07 smithi195 bash[72218]: debug 2024-03-07T03:58:07.291+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:07.565 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:07 smithi195 bash[72218]: debug 2024-03-07T03:58:07.467+0000 7f959afa7700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:58:07.565 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:07 smithi195 bash[72218]: debug 2024-03-07T03:58:07.467+0000 7f959afa7700 -1 osd.4 1102 *** Got signal Terminated *** 2024-03-07T03:58:07.565 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:07 smithi195 bash[72218]: debug 2024-03-07T03:58:07.467+0000 7f959afa7700 -1 osd.4 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:58:07.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:07 smithi195 bash[43128]: debug 2024-03-07T03:58:07.691+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:07.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:07 smithi195 bash[37711]: debug 2024-03-07T03:58:07.563+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:08.574 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:08 smithi195 bash[72218]: debug 2024-03-07T03:58:08.251+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:08.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:08 smithi195 bash[32082]: debug 2024-03-07T03:58:08.227+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:09.074 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:08 smithi195 bash[43128]: debug 2024-03-07T03:58:08.679+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:09.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:08 smithi195 bash[37711]: debug 2024-03-07T03:58:08.575+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:09.553 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:09 smithi195 bash[32082]: debug 2024-03-07T03:58:09.231+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:09.553 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:09 smithi195 bash[72218]: debug 2024-03-07T03:58:09.299+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:09.554 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:09 smithi195 bash[72218]: debug 2024-03-07T03:58:09.299+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.807569+0000 front 2024-03-07T03:57:46.807938+0000 (oldest deadline 2024-03-07T03:58:09.107410+0000) 2024-03-07T03:58:09.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:09 smithi195 bash[43128]: debug 2024-03-07T03:58:09.639+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:09.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:09 smithi195 bash[37711]: debug 2024-03-07T03:58:09.551+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:10.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:10 smithi148 bash[58229]: ts=2024-03-07T03:58:10.269Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:10.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:10 smithi148 bash[58229]: ts=2024-03-07T03:58:10.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:10.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:10 smithi148 bash[58229]: ts=2024-03-07T03:58:10.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:10.561 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:10 smithi195 bash[32082]: debug 2024-03-07T03:58:10.219+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:10.562 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:10 smithi195 bash[72218]: debug 2024-03-07T03:58:10.287+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:10.562 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:10 smithi195 bash[72218]: debug 2024-03-07T03:58:10.287+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.807569+0000 front 2024-03-07T03:57:46.807938+0000 (oldest deadline 2024-03-07T03:58:09.107410+0000) 2024-03-07T03:58:10.827 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:10 smithi195 bash[43128]: debug 2024-03-07T03:58:10.643+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:10.828 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:10 smithi195 bash[37711]: debug 2024-03-07T03:58:10.559+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:11.559 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:11 smithi195 bash[72218]: debug 2024-03-07T03:58:11.247+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:11.559 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:11 smithi195 bash[72218]: debug 2024-03-07T03:58:11.247+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.807569+0000 front 2024-03-07T03:57:46.807938+0000 (oldest deadline 2024-03-07T03:58:09.107410+0000) 2024-03-07T03:58:11.560 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:11 smithi195 bash[32082]: debug 2024-03-07T03:58:11.251+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:11.560 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:11 smithi195 bash[32082]: debug 2024-03-07T03:58:11.251+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:11.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:11 smithi195 bash[43128]: debug 2024-03-07T03:58:11.635+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:11.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:11 smithi195 bash[43128]: debug 2024-03-07T03:58:11.635+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:11.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:11 smithi195 bash[37711]: debug 2024-03-07T03:58:11.555+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:12.574 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[32082]: debug 2024-03-07T03:58:12.247+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:12.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[32082]: debug 2024-03-07T03:58:12.247+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:12.575 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[72218]: debug 2024-03-07T03:58:12.239+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.705794+0000 front 2024-03-07T03:57:35.706005+0000 (oldest deadline 2024-03-07T03:58:00.405425+0000) 2024-03-07T03:58:12.575 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[72218]: debug 2024-03-07T03:58:12.239+0000 7f959759b700 -1 osd.4 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.807569+0000 front 2024-03-07T03:57:46.807938+0000 (oldest deadline 2024-03-07T03:58:09.107410+0000) 2024-03-07T03:58:13.062 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[37711]: debug 2024-03-07T03:58:12.595+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:13.063 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[37711]: debug 2024-03-07T03:58:12.595+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:13.063 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[43128]: debug 2024-03-07T03:58:12.643+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:13.063 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:12 smithi195 bash[43128]: debug 2024-03-07T03:58:12.643+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:13.324 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[32082]: debug 2024-03-07T03:58:13.243+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:13.325 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[32082]: debug 2024-03-07T03:58:13.243+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:13.803 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:13 smithi148 bash[58229]: ts=2024-03-07T03:58:13.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:13.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:13 smithi148 bash[58229]: ts=2024-03-07T03:58:13.522Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:13.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:13 smithi148 bash[58229]: ts=2024-03-07T03:58:13.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:13.880 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[89897]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:58:13.881 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[37711]: debug 2024-03-07T03:58:13.619+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:13.881 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[37711]: debug 2024-03-07T03:58:13.619+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:13.881 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[43128]: debug 2024-03-07T03:58:13.647+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:13.881 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[43128]: debug 2024-03-07T03:58:13.647+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:14.213 INFO:journalctl@ceph.osd.4.smithi195.stdout:Mar 07 03:58:13 smithi195 bash[89961]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-4 2024-03-07T03:58:14.574 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:14 smithi195 bash[32082]: debug 2024-03-07T03:58:14.211+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:14.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:14 smithi195 bash[32082]: debug 2024-03-07T03:58:14.211+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:14.884 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:14 smithi195 bash[37711]: debug 2024-03-07T03:58:14.635+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:14.884 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:14 smithi195 bash[37711]: debug 2024-03-07T03:58:14.635+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:14.885 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:14 smithi195 bash[43128]: debug 2024-03-07T03:58:14.607+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:14.885 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:14 smithi195 bash[43128]: debug 2024-03-07T03:58:14.607+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:15.524 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:15 smithi195 bash[32082]: debug 2024-03-07T03:58:15.247+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:15.524 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:15 smithi195 bash[32082]: debug 2024-03-07T03:58:15.247+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:15.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:15 smithi195 bash[37711]: debug 2024-03-07T03:58:15.635+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:15.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:15 smithi195 bash[37711]: debug 2024-03-07T03:58:15.635+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:15.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:15 smithi195 bash[43128]: debug 2024-03-07T03:58:15.651+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:15.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:15 smithi195 bash[43128]: debug 2024-03-07T03:58:15.651+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:15.998 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.4.service' 2024-03-07T03:58:16.024 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:58:16.025 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-07T03:58:16.025 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-07T03:58:16.025 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.5 2024-03-07T03:58:16.290 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:16 smithi195 systemd[1]: Stopping Ceph osd.5 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:58:16.290 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[32082]: debug 2024-03-07T03:58:16.211+0000 7ff4afcdb700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:58:16.290 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[32082]: debug 2024-03-07T03:58:16.211+0000 7ff4afcdb700 -1 osd.5 1102 *** Got signal Terminated *** 2024-03-07T03:58:16.290 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[32082]: debug 2024-03-07T03:58:16.211+0000 7ff4afcdb700 -1 osd.5 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:58:16.574 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[32082]: debug 2024-03-07T03:58:16.287+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:16.574 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[32082]: debug 2024-03-07T03:58:16.287+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:17.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[37711]: debug 2024-03-07T03:58:16.675+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:17.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[37711]: debug 2024-03-07T03:58:16.675+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:17.075 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[43128]: debug 2024-03-07T03:58:16.607+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:17.075 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:16 smithi195 bash[43128]: debug 2024-03-07T03:58:16.607+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:17.636 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:17 smithi195 bash[43128]: debug 2024-03-07T03:58:17.571+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:17.636 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:17 smithi195 bash[43128]: debug 2024-03-07T03:58:17.571+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:17.637 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:17 smithi195 bash[32082]: debug 2024-03-07T03:58:17.335+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:17.637 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:17 smithi195 bash[32082]: debug 2024-03-07T03:58:17.335+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:17.637 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:17 smithi195 bash[32082]: debug 2024-03-07T03:58:17.335+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.626808+0000 front 2024-03-07T03:57:55.626865+0000 (oldest deadline 2024-03-07T03:58:16.726558+0000) 2024-03-07T03:58:18.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:17 smithi195 bash[37711]: debug 2024-03-07T03:58:17.635+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:18.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:17 smithi195 bash[37711]: debug 2024-03-07T03:58:17.635+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:18.648 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[32082]: debug 2024-03-07T03:58:18.351+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:18.649 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[32082]: debug 2024-03-07T03:58:18.351+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:18.649 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[32082]: debug 2024-03-07T03:58:18.351+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.626808+0000 front 2024-03-07T03:57:55.626865+0000 (oldest deadline 2024-03-07T03:58:16.726558+0000) 2024-03-07T03:58:18.649 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[43128]: debug 2024-03-07T03:58:18.551+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:18.649 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[43128]: debug 2024-03-07T03:58:18.551+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:18.649 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[43128]: debug 2024-03-07T03:58:18.551+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:19.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[37711]: debug 2024-03-07T03:58:18.647+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:19.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[37711]: debug 2024-03-07T03:58:18.647+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:19.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:18 smithi195 bash[37711]: debug 2024-03-07T03:58:18.647+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:19.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[32082]: debug 2024-03-07T03:58:19.319+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:19.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[32082]: debug 2024-03-07T03:58:19.319+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:19.575 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[32082]: debug 2024-03-07T03:58:19.319+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.626808+0000 front 2024-03-07T03:57:55.626865+0000 (oldest deadline 2024-03-07T03:58:16.726558+0000) 2024-03-07T03:58:19.575 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[43128]: debug 2024-03-07T03:58:19.559+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:19.575 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[43128]: debug 2024-03-07T03:58:19.559+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:19.575 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[43128]: debug 2024-03-07T03:58:19.559+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:20.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[37711]: debug 2024-03-07T03:58:19.679+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:20.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[37711]: debug 2024-03-07T03:58:19.679+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:20.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:19 smithi195 bash[37711]: debug 2024-03-07T03:58:19.679+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:20.553 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:20 smithi148 bash[58229]: ts=2024-03-07T03:58:20.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:20.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:20 smithi148 bash[58229]: ts=2024-03-07T03:58:20.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:20.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:20 smithi148 bash[58229]: ts=2024-03-07T03:58:20.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:20.665 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[32082]: debug 2024-03-07T03:58:20.347+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:35.921433+0000 front 2024-03-07T03:57:35.921459+0000 (oldest deadline 2024-03-07T03:58:00.621654+0000) 2024-03-07T03:58:20.665 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[32082]: debug 2024-03-07T03:58:20.347+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.424264+0000 front 2024-03-07T03:57:46.424227+0000 (oldest deadline 2024-03-07T03:58:11.124199+0000) 2024-03-07T03:58:20.665 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[32082]: debug 2024-03-07T03:58:20.347+0000 7ff4ac2d1700 -1 osd.5 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.626808+0000 front 2024-03-07T03:57:55.626865+0000 (oldest deadline 2024-03-07T03:58:16.726558+0000) 2024-03-07T03:58:20.665 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[43128]: debug 2024-03-07T03:58:20.511+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:20.666 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[43128]: debug 2024-03-07T03:58:20.511+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:20.666 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[43128]: debug 2024-03-07T03:58:20.511+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:21.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[37711]: debug 2024-03-07T03:58:20.663+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:21.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[37711]: debug 2024-03-07T03:58:20.663+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:21.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:20 smithi195 bash[37711]: debug 2024-03-07T03:58:20.663+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:21.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:21 smithi195 bash[37711]: debug 2024-03-07T03:58:21.663+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:21.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:21 smithi195 bash[37711]: debug 2024-03-07T03:58:21.663+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:21.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:21 smithi195 bash[37711]: debug 2024-03-07T03:58:21.663+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:21.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:21 smithi195 bash[43128]: debug 2024-03-07T03:58:21.543+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:21.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:21 smithi195 bash[43128]: debug 2024-03-07T03:58:21.543+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:21.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:21 smithi195 bash[43128]: debug 2024-03-07T03:58:21.543+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:22.698 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[90095]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-5 2024-03-07T03:58:22.698 INFO:journalctl@ceph.osd.5.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[90158]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-5 2024-03-07T03:58:22.698 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[43128]: debug 2024-03-07T03:58:22.527+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:22.699 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[43128]: debug 2024-03-07T03:58:22.527+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:22.699 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[43128]: debug 2024-03-07T03:58:22.527+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:23.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[37711]: debug 2024-03-07T03:58:22.695+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:23.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[37711]: debug 2024-03-07T03:58:22.695+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:23.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:22 smithi195 bash[37711]: debug 2024-03-07T03:58:22.695+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:23.803 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:23 smithi148 bash[58229]: ts=2024-03-07T03:58:23.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:23.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:23 smithi148 bash[58229]: ts=2024-03-07T03:58:23.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:23.804 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:23 smithi148 bash[58229]: ts=2024-03-07T03:58:23.524Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:23.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:23 smithi195 bash[43128]: debug 2024-03-07T03:58:23.543+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:23.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:23 smithi195 bash[43128]: debug 2024-03-07T03:58:23.543+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:23.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:23 smithi195 bash[43128]: debug 2024-03-07T03:58:23.543+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:23.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:23 smithi195 bash[37711]: debug 2024-03-07T03:58:23.691+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:23.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:23 smithi195 bash[37711]: debug 2024-03-07T03:58:23.691+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:23.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:23 smithi195 bash[37711]: debug 2024-03-07T03:58:23.691+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:24.741 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.5.service' 2024-03-07T03:58:24.767 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[37711]: debug 2024-03-07T03:58:24.671+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:24.767 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[37711]: debug 2024-03-07T03:58:24.671+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:24.767 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[37711]: debug 2024-03-07T03:58:24.671+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:24.768 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[43128]: debug 2024-03-07T03:58:24.499+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:24.768 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[43128]: debug 2024-03-07T03:58:24.499+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:24.768 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[43128]: debug 2024-03-07T03:58:24.499+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:24.769 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:58:24.770 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-07T03:58:24.770 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-07T03:58:24.770 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.6 2024-03-07T03:58:25.074 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:24 smithi195 systemd[1]: Stopping Ceph osd.6 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:58:25.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[37711]: debug 2024-03-07T03:58:24.943+0000 7f0ae0cd4700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:58:25.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[37711]: debug 2024-03-07T03:58:24.943+0000 7f0ae0cd4700 -1 osd.6 1102 *** Got signal Terminated *** 2024-03-07T03:58:25.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:24 smithi195 bash[37711]: debug 2024-03-07T03:58:24.943+0000 7f0ae0cd4700 -1 osd.6 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:58:25.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:25 smithi195 bash[37711]: debug 2024-03-07T03:58:25.655+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:25.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:25 smithi195 bash[37711]: debug 2024-03-07T03:58:25.655+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:25.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:25 smithi195 bash[37711]: debug 2024-03-07T03:58:25.655+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:25.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:25 smithi195 bash[43128]: debug 2024-03-07T03:58:25.535+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:25.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:25 smithi195 bash[43128]: debug 2024-03-07T03:58:25.535+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:25.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:25 smithi195 bash[43128]: debug 2024-03-07T03:58:25.535+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:26.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:26 smithi195 bash[37711]: debug 2024-03-07T03:58:26.623+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:26.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:26 smithi195 bash[37711]: debug 2024-03-07T03:58:26.623+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:26.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:26 smithi195 bash[37711]: debug 2024-03-07T03:58:26.623+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:26.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:26 smithi195 bash[43128]: debug 2024-03-07T03:58:26.567+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:26.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:26 smithi195 bash[43128]: debug 2024-03-07T03:58:26.567+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:26.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:26 smithi195 bash[43128]: debug 2024-03-07T03:58:26.567+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:27.824 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:27 smithi195 bash[37711]: debug 2024-03-07T03:58:27.595+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:27.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:27 smithi195 bash[37711]: debug 2024-03-07T03:58:27.595+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:27.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:27 smithi195 bash[37711]: debug 2024-03-07T03:58:27.595+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:27.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:27 smithi195 bash[43128]: debug 2024-03-07T03:58:27.567+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:27.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:27 smithi195 bash[43128]: debug 2024-03-07T03:58:27.567+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:27.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:27 smithi195 bash[43128]: debug 2024-03-07T03:58:27.567+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:27.826 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:27 smithi195 bash[43128]: debug 2024-03-07T03:58:27.567+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:29.074 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[43128]: debug 2024-03-07T03:58:28.579+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:29.075 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[43128]: debug 2024-03-07T03:58:28.579+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:29.075 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[43128]: debug 2024-03-07T03:58:28.579+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:29.075 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[43128]: debug 2024-03-07T03:58:28.579+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:29.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[37711]: debug 2024-03-07T03:58:28.575+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:29.075 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[37711]: debug 2024-03-07T03:58:28.575+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:29.076 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[37711]: debug 2024-03-07T03:58:28.575+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:29.076 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:28 smithi195 bash[37711]: debug 2024-03-07T03:58:28.575+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:02.487545+0000 front 2024-03-07T03:58:02.487349+0000 (oldest deadline 2024-03-07T03:58:28.387356+0000) 2024-03-07T03:58:29.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[43128]: debug 2024-03-07T03:58:29.539+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:29.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[43128]: debug 2024-03-07T03:58:29.539+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:29.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[43128]: debug 2024-03-07T03:58:29.539+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:29.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[43128]: debug 2024-03-07T03:58:29.539+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:29.825 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[37711]: debug 2024-03-07T03:58:29.539+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:36.782818+0000 front 2024-03-07T03:57:36.782526+0000 (oldest deadline 2024-03-07T03:58:00.882561+0000) 2024-03-07T03:58:29.826 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[37711]: debug 2024-03-07T03:58:29.539+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:46.684739+0000 front 2024-03-07T03:57:46.684810+0000 (oldest deadline 2024-03-07T03:58:12.584621+0000) 2024-03-07T03:58:29.826 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[37711]: debug 2024-03-07T03:58:29.539+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:53.086208+0000 front 2024-03-07T03:57:53.086163+0000 (oldest deadline 2024-03-07T03:58:18.385972+0000) 2024-03-07T03:58:29.826 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:29 smithi195 bash[37711]: debug 2024-03-07T03:58:29.539+0000 7f0adbac7700 -1 osd.6 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:02.487545+0000 front 2024-03-07T03:58:02.487349+0000 (oldest deadline 2024-03-07T03:58:28.387356+0000) 2024-03-07T03:58:30.553 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:30 smithi148 bash[58229]: ts=2024-03-07T03:58:30.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:30.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:30 smithi148 bash[58229]: ts=2024-03-07T03:58:30.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:30.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:30 smithi148 bash[58229]: ts=2024-03-07T03:58:30.273Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:30.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:30 smithi195 bash[43128]: debug 2024-03-07T03:58:30.507+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:30.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:30 smithi195 bash[43128]: debug 2024-03-07T03:58:30.507+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:30.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:30 smithi195 bash[43128]: debug 2024-03-07T03:58:30.507+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:30.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:30 smithi195 bash[43128]: debug 2024-03-07T03:58:30.507+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:31.324 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:30 smithi195 bash[90293]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-6 2024-03-07T03:58:31.325 INFO:journalctl@ceph.osd.6.smithi195.stdout:Mar 07 03:58:31 smithi195 bash[90359]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-6 2024-03-07T03:58:31.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:31 smithi195 bash[43128]: debug 2024-03-07T03:58:31.459+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:31.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:31 smithi195 bash[43128]: debug 2024-03-07T03:58:31.459+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:31.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:31 smithi195 bash[43128]: debug 2024-03-07T03:58:31.459+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:31.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:31 smithi195 bash[43128]: debug 2024-03-07T03:58:31.459+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:32.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[43128]: debug 2024-03-07T03:58:32.463+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:32.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[43128]: debug 2024-03-07T03:58:32.463+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:32.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[43128]: debug 2024-03-07T03:58:32.463+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:32.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[43128]: debug 2024-03-07T03:58:32.463+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:32.825 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[59134]: ts=2024-03-07T03:58:32.484Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:58:32.826 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[59134]: ts=2024-03-07T03:58:32.487Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:58:32.826 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[59134]: ts=2024-03-07T03:58:32.491Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:58:32.826 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[59134]: ts=2024-03-07T03:58:32.496Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:58:32.826 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:32 smithi195 bash[59134]: ts=2024-03-07T03:58:32.497Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.148:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.148:8765: connect: connection refused" 2024-03-07T03:58:33.142 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.6.service' 2024-03-07T03:58:33.169 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:58:33.170 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-07T03:58:33.170 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-07T03:58:33.170 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.7 2024-03-07T03:58:33.484 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 systemd[1]: Stopping Ceph osd.7 for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:58:33.485 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 bash[43128]: debug 2024-03-07T03:58:33.335+0000 7f21a3d9b700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-03-07T03:58:33.485 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 bash[43128]: debug 2024-03-07T03:58:33.335+0000 7f21a3d9b700 -1 osd.7 1102 *** Got signal Terminated *** 2024-03-07T03:58:33.485 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 bash[43128]: debug 2024-03-07T03:58:33.335+0000 7f21a3d9b700 -1 osd.7 1102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-07T03:58:33.803 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:33 smithi148 bash[58229]: ts=2024-03-07T03:58:33.523Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=2 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:33.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 bash[43128]: debug 2024-03-07T03:58:33.483+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:33.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 bash[43128]: debug 2024-03-07T03:58:33.483+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:33.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 bash[43128]: debug 2024-03-07T03:58:33.483+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:33.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:33 smithi195 bash[43128]: debug 2024-03-07T03:58:33.483+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:34.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:34 smithi195 bash[43128]: debug 2024-03-07T03:58:34.439+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:34.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:34 smithi195 bash[43128]: debug 2024-03-07T03:58:34.439+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:34.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:34 smithi195 bash[43128]: debug 2024-03-07T03:58:34.439+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:34.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:34 smithi195 bash[43128]: debug 2024-03-07T03:58:34.439+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:35.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:35 smithi195 bash[43128]: debug 2024-03-07T03:58:35.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:35.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:35 smithi195 bash[43128]: debug 2024-03-07T03:58:35.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:35.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:35 smithi195 bash[43128]: debug 2024-03-07T03:58:35.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:35.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:35 smithi195 bash[43128]: debug 2024-03-07T03:58:35.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:36.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:36 smithi195 bash[43128]: debug 2024-03-07T03:58:36.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:36.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:36 smithi195 bash[43128]: debug 2024-03-07T03:58:36.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:36.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:36 smithi195 bash[43128]: debug 2024-03-07T03:58:36.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:36.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:36 smithi195 bash[43128]: debug 2024-03-07T03:58:36.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:36.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:36 smithi195 bash[43128]: debug 2024-03-07T03:58:36.399+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.195:6804 osd.4 since back 2024-03-07T03:58:07.035482+0000 front 2024-03-07T03:58:07.035533+0000 (oldest deadline 2024-03-07T03:58:35.835825+0000) 2024-03-07T03:58:37.824 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:37 smithi195 bash[43128]: debug 2024-03-07T03:58:37.423+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6806 osd.0 since back 2024-03-07T03:57:37.829811+0000 front 2024-03-07T03:57:37.830091+0000 (oldest deadline 2024-03-07T03:58:05.430753+0000) 2024-03-07T03:58:37.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:37 smithi195 bash[43128]: debug 2024-03-07T03:58:37.423+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6814 osd.1 since back 2024-03-07T03:57:45.431677+0000 front 2024-03-07T03:57:45.431948+0000 (oldest deadline 2024-03-07T03:58:11.331403+0000) 2024-03-07T03:58:37.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:37 smithi195 bash[43128]: debug 2024-03-07T03:58:37.423+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6822 osd.2 since back 2024-03-07T03:57:55.333475+0000 front 2024-03-07T03:57:55.333556+0000 (oldest deadline 2024-03-07T03:58:17.633451+0000) 2024-03-07T03:58:37.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:37 smithi195 bash[43128]: debug 2024-03-07T03:58:37.423+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.148:6830 osd.3 since back 2024-03-07T03:58:01.735004+0000 front 2024-03-07T03:58:01.735093+0000 (oldest deadline 2024-03-07T03:58:27.034759+0000) 2024-03-07T03:58:37.825 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:37 smithi195 bash[43128]: debug 2024-03-07T03:58:37.423+0000 7f219db8c700 -1 osd.7 1102 heartbeat_check: no reply from 172.21.15.195:6804 osd.4 since back 2024-03-07T03:58:07.035482+0000 front 2024-03-07T03:58:07.035533+0000 (oldest deadline 2024-03-07T03:58:35.835825+0000) 2024-03-07T03:58:39.685 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:39 smithi195 bash[90490]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-7 2024-03-07T03:58:39.686 INFO:journalctl@ceph.osd.7.smithi195.stdout:Mar 07 03:58:39 smithi195 bash[90554]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-osd-7 2024-03-07T03:58:40.553 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:40 smithi148 bash[58229]: ts=2024-03-07T03:58:40.271Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:40.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:40 smithi148 bash[58229]: ts=2024-03-07T03:58:40.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:40.554 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:40 smithi148 bash[58229]: ts=2024-03-07T03:58:40.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:41.549 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@osd.7.service' 2024-03-07T03:58:41.576 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:58:41.577 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-07T03:58:41.577 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-03-07T03:58:41.577 DEBUG:teuthology.orchestra.run.smithi195:> sudo systemctl stop ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@prometheus.a 2024-03-07T03:58:41.808 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 systemd[1]: Stopping Ceph prometheus.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:58:41.809 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.766Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:58:41.809 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.766Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-07T03:58:41.809 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.766Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-07T03:58:41.810 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.766Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-07T03:58:41.810 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.766Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-07T03:58:41.810 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.766Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-07T03:58:41.810 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.766Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-07T03:58:41.810 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.767Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-07T03:58:41.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.767Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-07T03:58:41.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.768Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-07T03:58:41.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.768Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-07T03:58:41.811 INFO:journalctl@ceph.prometheus.a.smithi195.stdout:Mar 07 03:58:41 smithi195 bash[59134]: ts=2024-03-07T03:58:41.768Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-07T03:58:42.600 DEBUG:teuthology.orchestra.run.smithi195:> sudo pkill -f 'journalctl -f -n 0 -u ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@prometheus.a.service' 2024-03-07T03:58:42.621 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-07T03:58:42.622 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-03-07T03:58:42.622 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 --force --keep-logs 2024-03-07T03:58:42.744 INFO:teuthology.orchestra.run.smithi148.stdout:Deleting cluster with fsid: b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:58:47.580 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:47 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:47.580 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:47 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:47.945 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:47 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:47.945 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:47 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.209 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:47 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.209 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:47 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.507 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.507 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.507 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.507 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.760 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:48.761 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:49.053 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:49.054 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:49.054 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:48 smithi148 systemd[1]: Stopping Ceph node-exporter.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:58:50.053 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:49 smithi148 bash[178235]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-node-exporter-a 2024-03-07T03:58:50.053 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:49 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:58:50.054 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:49 smithi148 bash[178295]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-node-exporter-a 2024-03-07T03:58:50.054 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:49 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-07T03:58:50.054 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:49 smithi148 systemd[1]: Stopped Ceph node-exporter.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:58:50.338 INFO:journalctl@ceph.node-exporter.a.smithi148.stdout:Mar 07 03:58:50 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:50.339 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:50.339 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 systemd[1]: Stopping Ceph alertmanager.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:58:50.339 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 bash[58229]: ts=2024-03-07T03:58:50.273Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:50.339 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 bash[58229]: ts=2024-03-07T03:58:50.273Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi148.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.148:8443: connect: connection refused" 2024-03-07T03:58:50.339 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 bash[58229]: ts=2024-03-07T03:58:50.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi195.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.195:8443: connect: connection refused" 2024-03-07T03:58:50.339 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 bash[58229]: ts=2024-03-07T03:58:50.297Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-07T03:58:51.157 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 bash[178356]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:58:51.158 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:50 smithi148 bash[178417]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-alertmanager-a 2024-03-07T03:58:51.158 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:51 smithi148 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-07T03:58:51.158 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:51 smithi148 systemd[1]: Stopped Ceph alertmanager.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:58:51.553 INFO:journalctl@ceph.alertmanager.a.smithi148.stdout:Mar 07 03:58:51 smithi148 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:54.675 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 --force --keep-logs 2024-03-07T03:58:54.795 INFO:teuthology.orchestra.run.smithi195.stdout:Deleting cluster with fsid: b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:58:59.761 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:58:59 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:58:59.761 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:58:59 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.065 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:58:59 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.065 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.065 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:58:59 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.065 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.324 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.325 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.697 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.697 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.980 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.980 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.981 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:00.981 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:00 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:01.324 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:01 smithi195 systemd[1]: Stopping Ceph node-exporter.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:59:01.987 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:01 smithi195 bash[91014]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-node-exporter-b 2024-03-07T03:59:01.987 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:01 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:59:01.987 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:01 smithi195 bash[91075]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-node-exporter-b 2024-03-07T03:59:01.987 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:01 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-07T03:59:01.987 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:01 smithi195 systemd[1]: Stopped Ceph node-exporter.b for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:59:02.243 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:01 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:02.243 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:02 smithi195 systemd[1]: Stopping Ceph grafana.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715... 2024-03-07T03:59:02.244 INFO:journalctl@ceph.node-exporter.b.smithi195.stdout:Mar 07 03:59:02 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:03.074 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:02 smithi195 bash[91138]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-grafana-a 2024-03-07T03:59:03.075 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:02 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-03-07T03:59:03.075 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:02 smithi195 bash[91200]: Error response from daemon: No such container: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715-grafana-a 2024-03-07T03:59:03.075 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:02 smithi195 systemd[1]: ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-07T03:59:03.075 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:02 smithi195 systemd[1]: Stopped Ceph grafana.a for b9c09ac8-dc2d-11ee-95c6-87774f69a715. 2024-03-07T03:59:03.574 INFO:journalctl@ceph.grafana.a.smithi195.stdout:Mar 07 03:59:03 smithi195 systemd[1]: /etc/systemd/system/ceph-b9c09ac8-dc2d-11ee-95c6-87774f69a715@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-03-07T03:59:14.766 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:59:14.780 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-07T03:59:14.793 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-07T03:59:14.795 DEBUG:teuthology.misc:Transferring archived files from smithi148:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780/remote/smithi148/crash 2024-03-07T03:59:14.796 DEBUG:teuthology.orchestra.run.smithi148:> sudo tar c -f - -C /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/crash -- . 2024-03-07T03:59:14.838 INFO:teuthology.orchestra.run.smithi148.stderr:tar: /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/crash: Cannot open: No such file or directory 2024-03-07T03:59:14.839 INFO:teuthology.orchestra.run.smithi148.stderr:tar: Error is not recoverable: exiting now 2024-03-07T03:59:14.841 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780/remote/smithi195/crash 2024-03-07T03:59:14.842 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/crash -- . 2024-03-07T03:59:14.854 INFO:teuthology.orchestra.run.smithi195.stderr:tar: /var/lib/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/crash: Cannot open: No such file or directory 2024-03-07T03:59:14.854 INFO:teuthology.orchestra.run.smithi195.stderr:tar: Error is not recoverable: exiting now 2024-03-07T03:59:14.857 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-07T03:59:14.857 DEBUG:teuthology.orchestra.run.smithi148:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-07T03:59:14.897 INFO:teuthology.orchestra.run.smithi148.stdout:2024-03-07T03:00:59.249514+0000 mon.a (mon.0) 185 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2024-03-07T03:59:14.898 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-07T03:59:14.898 INFO:tasks.cephadm:Compressing logs... 2024-03-07T03:59:14.898 DEBUG:teuthology.orchestra.run.smithi148:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-07T03:59:14.941 DEBUG:teuthology.orchestra.run.smithi195:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-07T03:59:14.949 INFO:teuthology.orchestra.run.smithi148.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-03-07T03:59:14.954 INFO:teuthology.orchestra.run.smithi195.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-03-07T03:59:14.955 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-07T03:59:14.956 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.y.log 2024-03-07T03:59:14.957 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.0.log 2024-03-07T03:59:14.957 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.2.log 2024-03-07T03:59:14.957 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.a.log 2024-03-07T03:59:14.958 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log 2024-03-07T03:59:14.958 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.3.log 2024-03-07T03:59:14.959 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi148.qizikc.log 2024-03-07T03:59:14.960 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.1.log 2024-03-07T03:59:14.960 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi148.qizikc.log: 90.7% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log.gz 2024-03-07T03:59:14.960 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.log 2024-03-07T03:59:14.961 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.1.log: 87.0% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi148.qizikc.log.gz 2024-03-07T03:59:14.961 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log 2024-03-07T03:59:14.963 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.log: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-07T03:59:14.964 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-07T03:59:14.964 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.c.log 2024-03-07T03:59:14.965 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.7.log 2024-03-07T03:59:14.966 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.6.log 2024-03-07T03:59:14.966 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log 2024-03-07T03:59:14.967 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.5.log 2024-03-07T03:59:14.967 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.6.log: /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.x.log 2024-03-07T03:59:14.967 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.log 2024-03-07T03:59:14.967 INFO:teuthology.orchestra.run.smithi195.stderr: 83.8% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.cephadm.log.gz 2024-03-07T03:59:14.969 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi195.gtlmjn.log 2024-03-07T03:59:14.969 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log 2024-03-07T03:59:14.969 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi195.gtlmjn.log: gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log 2024-03-07T03:59:14.970 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log: 85.5% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mds.foo.smithi195.gtlmjn.log.gz 2024-03-07T03:59:14.971 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.b.log 2024-03-07T03:59:14.976 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log: 84.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-07T03:59:14.977 INFO:teuthology.orchestra.run.smithi195.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.4.log 2024-03-07T03:59:14.978 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log: 92.3% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.log.gz 2024-03-07T03:59:14.978 INFO:teuthology.orchestra.run.smithi148.stderr:gzip -5 --verbose -- /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log 2024-03-07T03:59:14.979 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.b.log: 90.6% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log.gz 2024-03-07T03:59:14.979 INFO:teuthology.orchestra.run.smithi195.stderr: 86.4% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.log.gz 2024-03-07T03:59:14.986 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.c.log: 91.2% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log.gz 2024-03-07T03:59:14.987 INFO:teuthology.orchestra.run.smithi195.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.4.log: 91.4% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-volume.log.gz 2024-03-07T03:59:14.997 INFO:teuthology.orchestra.run.smithi148.stderr:/var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log: 94.3% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph.audit.log.gz 2024-03-07T03:59:15.029 INFO:teuthology.orchestra.run.smithi195.stderr: 89.8% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.x.log.gz 2024-03-07T03:59:15.645 INFO:teuthology.orchestra.run.smithi148.stderr: 89.5% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mgr.y.log.gz 2024-03-07T03:59:16.144 INFO:teuthology.orchestra.run.smithi195.stderr: 92.6% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.b.log.gz 2024-03-07T03:59:16.278 INFO:teuthology.orchestra.run.smithi148.stderr: 92.2% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.c.log.gz 2024-03-07T03:59:17.178 INFO:teuthology.orchestra.run.smithi148.stderr: 90.7% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-mon.a.log.gz 2024-03-07T03:59:22.937 INFO:teuthology.orchestra.run.smithi195.stderr: 94.3% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.5.log.gz 2024-03-07T03:59:23.767 INFO:teuthology.orchestra.run.smithi195.stderr: 94.4% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.7.log.gz 2024-03-07T03:59:24.118 INFO:teuthology.orchestra.run.smithi195.stderr: 94.5% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.4.log.gz 2024-03-07T03:59:24.177 INFO:teuthology.orchestra.run.smithi148.stderr: 94.4% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.2.log.gz 2024-03-07T03:59:25.046 INFO:teuthology.orchestra.run.smithi148.stderr: 94.5% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.0.log.gz 2024-03-07T03:59:25.358 INFO:teuthology.orchestra.run.smithi195.stderr: 94.7% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.6.log.gz 2024-03-07T03:59:25.359 INFO:teuthology.orchestra.run.smithi195.stderr: 2024-03-07T03:59:25.359 INFO:teuthology.orchestra.run.smithi195.stderr:real 0m10.414s 2024-03-07T03:59:25.359 INFO:teuthology.orchestra.run.smithi195.stderr:user 0m36.211s 2024-03-07T03:59:25.360 INFO:teuthology.orchestra.run.smithi195.stderr:sys 0m1.398s 2024-03-07T03:59:26.139 INFO:teuthology.orchestra.run.smithi148.stderr: 94.9% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.1.log.gz 2024-03-07T03:59:26.777 INFO:teuthology.orchestra.run.smithi148.stderr: 94.9% -- replaced with /var/log/ceph/b9c09ac8-dc2d-11ee-95c6-87774f69a715/ceph-osd.3.log.gz 2024-03-07T03:59:26.779 INFO:teuthology.orchestra.run.smithi148.stderr: 2024-03-07T03:59:26.779 INFO:teuthology.orchestra.run.smithi148.stderr:real 0m11.836s 2024-03-07T03:59:26.780 INFO:teuthology.orchestra.run.smithi148.stderr:user 0m44.884s 2024-03-07T03:59:26.780 INFO:teuthology.orchestra.run.smithi148.stderr:sys 0m1.710s 2024-03-07T03:59:26.780 INFO:tasks.cephadm:Archiving logs... 2024-03-07T03:59:26.780 DEBUG:teuthology.misc:Transferring archived files from smithi148:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780/remote/smithi148/log 2024-03-07T03:59:26.781 DEBUG:teuthology.orchestra.run.smithi148:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-07T03:59:31.954 DEBUG:teuthology.misc:Transferring archived files from smithi195:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780/remote/smithi195/log 2024-03-07T03:59:31.955 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-07T03:59:36.171 INFO:tasks.cephadm:Removing cluster... 2024-03-07T03:59:36.171 DEBUG:teuthology.orchestra.run.smithi148:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 --force 2024-03-07T03:59:36.294 INFO:teuthology.orchestra.run.smithi148.stdout:Deleting cluster with fsid: b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:59:37.676 DEBUG:teuthology.orchestra.run.smithi195:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid b9c09ac8-dc2d-11ee-95c6-87774f69a715 --force 2024-03-07T03:59:37.798 INFO:teuthology.orchestra.run.smithi195.stdout:Deleting cluster with fsid: b9c09ac8-dc2d-11ee-95c6-87774f69a715 2024-03-07T03:59:39.214 INFO:tasks.cephadm:Removing cephadm ... 2024-03-07T03:59:39.214 DEBUG:teuthology.orchestra.run.smithi148:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-07T03:59:39.220 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-07T03:59:39.225 INFO:tasks.cephadm:Teardown complete 2024-03-07T03:59:39.226 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-03-07T03:59:39.236 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-03-07T03:59:39.237 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-07T03:59:39.266 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-07T03:59:39.329 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-07T03:59:39.341 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-07T03:59:39.341 DEBUG:teuthology.orchestra.run.smithi148:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T03:59:39.344 DEBUG:teuthology.orchestra.run.smithi195:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-07T03:59:39.359 INFO:teuthology.orchestra.run.smithi148.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T03:59:39.359 INFO:teuthology.orchestra.run.smithi148.stdout:============================================================================== 2024-03-07T03:59:39.359 INFO:teuthology.orchestra.run.smithi148.stdout:*hv01.front.sepi 44.4.53.2 2 u 49 64 377 0.075 -4.329 0.098 2024-03-07T03:59:39.359 INFO:teuthology.orchestra.run.smithi148.stdout:+hv02.front.sepi 63.231.80.2 3 u 49 64 377 0.098 +0.777 0.083 2024-03-07T03:59:39.359 INFO:teuthology.orchestra.run.smithi148.stdout:+hv03.front.sepi 74.6.168.72 3 u 48 128 377 0.082 -1.010 0.103 2024-03-07T03:59:39.359 INFO:teuthology.orchestra.run.smithi148.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-03-07T03:59:39.361 INFO:teuthology.orchestra.run.smithi195.stdout: remote refid st t when poll reach delay offset jitter 2024-03-07T03:59:39.361 INFO:teuthology.orchestra.run.smithi195.stdout:============================================================================== 2024-03-07T03:59:39.361 INFO:teuthology.orchestra.run.smithi195.stdout:*hv01.front.sepi 44.4.53.2 2 u 54 128 377 0.137 -4.885 0.388 2024-03-07T03:59:39.362 INFO:teuthology.orchestra.run.smithi195.stdout:+hv02.front.sepi 63.231.80.2 3 u 118 128 377 0.121 +0.049 0.320 2024-03-07T03:59:39.362 INFO:teuthology.orchestra.run.smithi195.stdout:+hv03.front.sepi 74.6.168.72 3 u 114 128 377 0.107 -1.097 0.656 2024-03-07T03:59:39.362 INFO:teuthology.orchestra.run.smithi195.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-03-07T03:59:39.362 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-07T03:59:39.374 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-07T03:59:39.375 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-07T03:59:39.390 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-07T03:59:39.405 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-07T03:59:39.421 INFO:teuthology.task.internal:Duration was 4723.207255 seconds 2024-03-07T03:59:39.421 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-07T03:59:39.437 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-07T03:59:39.437 DEBUG:teuthology.orchestra.run.smithi148:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-07T03:59:39.440 DEBUG:teuthology.orchestra.run.smithi195:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-07T03:59:39.477 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-07T03:59:39.477 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi148.front.sepia.ceph.com 2024-03-07T03:59:39.477 DEBUG:teuthology.orchestra.run.smithi148:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-07T03:59:39.525 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi195.front.sepia.ceph.com 2024-03-07T03:59:39.526 DEBUG:teuthology.orchestra.run.smithi195:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-07T03:59:39.536 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-07T03:59:39.536 DEBUG:teuthology.orchestra.run.smithi148:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-07T03:59:39.573 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-07T03:59:39.649 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-07T03:59:39.650 DEBUG:teuthology.orchestra.run.smithi148:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-07T03:59:39.651 DEBUG:teuthology.orchestra.run.smithi195:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-07T03:59:39.883 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-07T03:59:39.894 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-07T03:59:39.894 DEBUG:teuthology.orchestra.run.smithi148:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-07T03:59:39.906 DEBUG:teuthology.orchestra.run.smithi195:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-07T03:59:39.919 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-07T03:59:39.932 DEBUG:teuthology.orchestra.run.smithi148:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:59:39.949 DEBUG:teuthology.orchestra.run.smithi195:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:59:39.958 INFO:teuthology.orchestra.run.smithi148.stdout:kernel.core_pattern = core 2024-03-07T03:59:39.973 INFO:teuthology.orchestra.run.smithi195.stdout:kernel.core_pattern = core 2024-03-07T03:59:40.083 DEBUG:teuthology.orchestra.run.smithi148:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:59:40.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T03:59:40.088 DEBUG:teuthology.orchestra.run.smithi195:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-07T03:59:40.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-07T03:59:40.131 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-07T03:59:40.142 INFO:teuthology.task.internal:Transferring archived files... 2024-03-07T03:59:40.143 DEBUG:teuthology.misc:Transferring archived files from smithi148:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780/remote/smithi148 2024-03-07T03:59:40.143 DEBUG:teuthology.orchestra.run.smithi148:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-07T03:59:40.175 DEBUG:teuthology.misc:Transferring archived files from smithi195:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780/remote/smithi195 2024-03-07T03:59:40.176 DEBUG:teuthology.orchestra.run.smithi195:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-07T03:59:40.202 INFO:teuthology.task.internal:Removing archive directory... 2024-03-07T03:59:40.202 DEBUG:teuthology.orchestra.run.smithi148:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-07T03:59:40.213 DEBUG:teuthology.orchestra.run.smithi195:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-07T03:59:40.247 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-07T03:59:40.259 INFO:teuthology.task.internal:Not uploading archives. 2024-03-07T03:59:40.259 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-07T03:59:40.374 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-07T03:59:40.375 DEBUG:teuthology.orchestra.run.smithi148:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-07T03:59:40.377 DEBUG:teuthology.orchestra.run.smithi195:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-07T03:59:40.381 INFO:teuthology.orchestra.run.smithi148.stdout: 658467 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 7 03:59 /home/ubuntu/cephtest 2024-03-07T03:59:40.384 INFO:teuthology.orchestra.run.smithi195.stdout: 658642 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 7 03:59 /home/ubuntu/cephtest 2024-03-07T03:59:40.385 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-07T03:59:40.396 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-07T03:59:40.428 INFO:teuthology.nuke:Checking targets against current locks 2024-03-07T03:59:40.456 DEBUG:teuthology.nuke:shortname: smithi148 2024-03-07T03:59:40.456 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-07T03:59:40.483 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi148.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:27.937163', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBA4zgKKJtKEHB2ABRlRQ0xVhJ18MtIh0fh6AfF4DkHvr0jeQV9eHB44QqGWqgac5b4r93UhBb3Es3CRRsyen9IQ='} 2024-03-07T03:59:40.484 DEBUG:teuthology.nuke:shortname: smithi195 2024-03-07T03:59:40.484 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-07T03:59:40.507 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi195.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-07_02:25:02-upgrade:reef-x-main-distro-default-smithi/7583780', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-07 02:26:27.938540', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBM69fqkVao/QzEKb9dWEj4UEZOWwl5KaihGNOJXzaIZxOxle2qemPgCKNrf1fsSPG2TOFVC33onxYsRglMh532g='} 2024-03-07T03:59:40.541 INFO:teuthology.orchestra.console.smithi148:Power off 2024-03-07T03:59:40.541 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-07T03:59:40.565 DEBUG:teuthology.orchestra.console.smithi148:power off output: Chassis Power Control: Down/Off 2024-03-07T03:59:40.566 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:59:40.581 INFO:teuthology.orchestra.console.smithi195:Power off 2024-03-07T03:59:40.581 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-07T03:59:40.590 DEBUG:teuthology.orchestra.console.smithi148:check power output: Chassis Power is on 2024-03-07T03:59:40.603 DEBUG:teuthology.orchestra.console.smithi195:power off output: Chassis Power Control: Down/Off 2024-03-07T03:59:40.604 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:59:40.627 DEBUG:teuthology.orchestra.console.smithi195:check power output: Chassis Power is on 2024-03-07T03:59:44.591 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:59:44.628 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:59:44.703 DEBUG:teuthology.orchestra.console.smithi148:check power output: Chassis Power is on 2024-03-07T03:59:44.739 DEBUG:teuthology.orchestra.console.smithi195:check power output: Chassis Power is on 2024-03-07T03:59:48.708 DEBUG:teuthology.orchestra.console.smithi148:pexpect command: ipmitool -H smithi148.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:59:48.740 DEBUG:teuthology.orchestra.console.smithi195:pexpect command: ipmitool -H smithi195.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-07T03:59:48.818 DEBUG:teuthology.orchestra.console.smithi148:check power output: Chassis Power is off 2024-03-07T03:59:48.849 DEBUG:teuthology.orchestra.console.smithi195:check power output: Chassis Power is off 2024-03-07T03:59:48.919 INFO:teuthology.orchestra.console.smithi148:Power off completed 2024-03-07T03:59:48.950 INFO:teuthology.orchestra.console.smithi195:Power off completed 2024-03-07T03:59:49.055 INFO:teuthology.run:Summary data: description: upgrade:reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 4723.207255363464 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e6521ef688a34be887461dfa2bcae1a1 status: fail success: false 2024-03-07T03:59:49.055 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-07T03:59:49.136 INFO:teuthology.run:FAIL